diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 11c15352a..7a83066ed 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -59,6 +59,8 @@ These are the canonical homes. Do not reintroduce private copies elsewhere. | Presentation-layer DI composition root | `AppServiceRegistration` (root `ServiceProvider`, owned by `App`) | authoritative | | Settings snapshot read + batched save + non-echoing change notification | `ISettingsStore` | authoritative | | Settings page load/persist view logic | `SettingsPageViewModel` | authoritative | +| Managed-local listener provenance and strong-credential authorization | `ManagedLocalGatewayPortProvenanceService` | authoritative | +| Managed-local automatic repair eligibility and orchestration | `ManagedLocalGatewayAutoRepairMonitor` + `ManagedLocalGatewayRepairCoordinator` | authoritative | | Capability UI metadata | `NodeCapabilityUiCatalog` (planned) | planned | | Capability registration/gating | `NodeCapabilityRegistrationPolicy` (planned) | planned | | Local MCP exposure policy | `McpCapabilityPolicy` (planned) | planned | @@ -117,6 +119,9 @@ leading and trailing pipe. Columns, in order: | wsl-posix-quoting | authoritative | OpenClaw.SetupEngine/SetupSteps.cs | ad hoc ShellEscape with divergent wrap semantics | WslShellQuoting | - | WSL command lines use POSIX single-quote quoting via WslShellQuoting not cmd/PowerShell quoting | WslShellQuotingTests.QuotePosixSingleQuote_WrapsAndEscapesEmbeddedQuote | behavioral | when no code builds WSL command lines outside WslShellQuoting | | setup-shellescape-closed | closed | src/OpenClaw.SetupEngine/SetupSteps.cs | private ShellEscape helpers with divergent wrap semantics | WslShellQuoting | - | SetupSteps builds WSL command lines only via WslShellQuoting; no local ShellEscape helper | SetupStepsShellEscapeClosureTests.SetupSteps_DoesNotReintroduce_PrivateShellEscape | source-shape | when SetupSteps.cs no longer builds any WSL command strings | | wsl-distro-install-path | authoritative | OpenClaw.SetupEngine/SetupSteps.cs | inline Path.Combine wsl distro install-path derivation | DistroInstallPathPolicy | - | new installs use the strict supported name grammar; teardown accepts only unambiguous single-segment names whose canonical path is an immediate child of LocalDataDir\wsl with no aliases, case or Unicode collisions, or reparse points at the root or child | SetupStepsTests.DistroInstallPathPolicy_ResolvesImmediateChild | behavioral | - | +| managed-local-provenance | authoritative | scattered connection, setup, browser, and reconnect call sites | implicit loopback trust and duplicated strong-credential listener checks | ManagedLocalGatewayPortProvenanceService | callers request inspection, authorization, or conflict repair only | unknown or changed listener owners never receive strong credentials or destructive remediation | ManagedLocalGatewayPortProvenanceServiceTests.InteractiveCredentialGate_ExpectedCacheThenOwnerChanges_FailsClosed | behavioral | - | +| managed-local-repair | authoritative | src/OpenClaw.Tray.WinUI/App.xaml.cs and direct reconnect callbacks | repair eligibility, restart budgets, port remediation, and reconnect verification | ManagedLocalGatewayAutoRepairMonitor + ManagedLocalGatewayRepairCoordinator | App composition and dependency callbacks only | explicit disconnect and gateway switches abort repair before restart or reconnect | ManagedLocalGatewayRepairCoordinatorTests.UserDisconnectedIntent_AbortsBeforeProbeOrRestart | behavioral | - | +| app-managed-local-repair-closed | closed | src/OpenClaw.Tray.WinUI/App.xaml.cs | managed-local repair loops, probing, restart budgeting, and verification implementation | ManagedLocalGatewayAutoRepairMonitor + ManagedLocalGatewayRepairCoordinator | service construction, callback adapters, and lifetime wiring only | App remains the composition root and does not regain repair implementation | AppRefactorContractTests.ManagedLocalGatewayRepair_StaysDelegatedToDedicatedOwners | source-shape | when App no longer constructs the managed-local repair services directly | | app-window-manager | planned | src/OpenClaw.Tray.WinUI/App.xaml.cs | window creation/show/hide/shutdown | IWindowManager | composition/delegation only | startup/shutdown ordering deterministic; disposed once | none | review-only | extracted in Phase 3 | | app-tray-controller | planned | src/OpenClaw.Tray.WinUI/App.xaml.cs | tray icon/menu/action routing | ITrayController | composition/delegation only | tray actions route unchanged | none | review-only | extracted in Phase 3 | | app-activation-router | planned | src/OpenClaw.Tray.WinUI/App.xaml.cs | deep-link/toast/single-instance activation | IActivationRouter | composition/delegation only | activation routes land on the same UI/actions; current-user pipe security preserved | none | review-only | extracted in Phase 3 | diff --git a/docs/CONNECTION_ARCHITECTURE.md b/docs/CONNECTION_ARCHITECTURE.md index facd11278..d0e656f6b 100644 --- a/docs/CONNECTION_ARCHITECTURE.md +++ b/docs/CONNECTION_ARCHITECTURE.md @@ -194,6 +194,24 @@ Node credential precedence follows the same invariant with a distinct stored tok **`InteractiveGatewayCredentialResolver`** resolves credentials for HTTP surfaces (chat URL `?token=` auth). It **prefers SharedGatewayToken** over DeviceToken because HTTP endpoints expect the shared token, not the per-device WebSocket token. Browser proxy diagnostics should treat the missing shared token as a browser-control caveat, not as proof that the operator or node gateway connection is disconnected. +## Self-recovery and automatic local-gateway repair + +Two orthogonal self-healing behaviors keep the connection reliable without dead-ending the user: + +### Stale device-token self-recovery (operator + node) + +The gateway may reject a stored device token with the structured code `AUTH_DEVICE_TOKEN_MISMATCH` (a rotated/revoked/replaced device token) — distinct from a wrong *shared* token. `GatewayErrorClassifier` is the single classifier for this: `ClassifyWithCode(message, ...codes)` inspects the structured `error.code`/`error.details.code` **before** the textual heuristic and returns the exact `GatewayErrorKind.DeviceTokenMismatch`, keeping a stale *device* token (auto-recoverable) separate from a wrong *shared* token (`Auth`, not device-recoverable). Broad `GatewayErrorKind.TokenDrift` remains a manual re-pair signal for UI copy. + +On a device-token mismatch, the manager clears **only the rejected role's** device token and reconnects, letting `CredentialResolver` fall back to the same record's `SharedGatewayToken` (preferred) or `BootstrapToken`. This kills the post-setup "need a new token" dead end (setup clears the bootstrap token once pairing is durable, but the shared token remains). Operator recovery runs in `TryScheduleOperatorTokenRecovery`; node recovery is driven off the node client's classified `INodeConnectorTelemetryEvents.ConnectionFailure(GatewayErrorKind)` — the manager's `OnNodeConnectionFailure` queues `HandleNodeDeviceTokenMismatchAsync` off the connector's dispatch lock (capturing lifecycle+node generations at fire time and re-checking `IsCurrentNodeAttempt` before/after the transition semaphore). A per-gateway, per-role attempt guard (reset on handshake success / node pairing) prevents clear→reconnect→mismatch loops. + +**Security — trust gate and endpoint provenance.** Clearing a device token downgrades to the more powerful shared/bootstrap credential, so `IsRecoverySafeEndpoint` restricts recovery to trusted endpoints: an owned SSH tunnel, a validated TLS (`wss`/`https`) endpoint, or — for a setup-managed WSL loopback gateway — a listener proven by `ManagedLocalGatewayPortProvenanceService` to be the Windows WSL relay. Loopback is not treated as identity by itself: an unknown listener or a proven obsolete native OpenClaw gateway blocks fallback, so a wrong local process cannot return a device-token mismatch to induce disclosure of the shared credential. A plain `ws://` remote endpoint is never eligible. + +### Automatic managed-local WSL gateway repair (tray) + +For an app-owned setup-managed local WSL gateway (`WslKeepAlivePolicy.IsSetupManagedLocalRecord` — never SSH/remote/ambiguous-localhost), the tray owns process supervision, keeping it out of the connection layer. `ManagedLocalGatewayAutoRepairMonitor` watches the operator connection and, when it is positively transport-unreachable (`GatewayErrorKind.Network`/`Server`, plus a cold-start `Connecting` state with no failure yet; never unknown/auth/pairing/rate-limit/scope/TLS/tunnel/token-drift) for a sustained window, invokes `ManagedLocalGatewayRepairCoordinator`. A typed `LocalPortConflict` is also repairable because its remediation is provenance-gated rather than a blind process restart. The monitor honors a **startup grace** (so a slow WSL cold start is not interrupted), a per-gateway unhealthy threshold and cooldown, a manager-owned explicit disconnect/stop intent, and a settings **kill switch** (`SettingsData.EnableManagedLocalGatewayAutoRepair`, default on). + +`ManagedLocalGatewayRepairCoordinator` **probes before it restarts**: if the gateway is already reachable it just reconnects (the macOS "attach" path); only a genuinely-down gateway triggers a WSL distro restart (via `WslGatewayController`), a keepalive re-arm (`WslGatewayKeepAliveService.TryEnsureAsync`), and a reconnect. For the native-vs-WSL collision case, `ManagedLocalGatewayPortProvenanceService` classifies listeners by address and proves process command line plus scheduled-task/profile lineage. It automatically disables/stops only a fully proven obsolete native OpenClaw gateway; an unknown listener is never killed and produces precise `LocalPortConflict` diagnostics. The shared lifecycle lease serializes that destructive work with manual WSL actions. Reconnect is **gateway-pinned, intent-aware, and cancellable** (`GatewayConnectionManager.ReconnectIfCurrentAsync(gatewayId, ct)`), so gateway switches, explicit Disconnect/Stop, and shutdown always win. Repair is single-flight, verifies success by a real operator connection to the same gateway, is per-gateway restart-budget-bounded, and never reads or logs credentials. + ## Client instance lifecycle **Operator client** (`OpenClawGatewayClient`): Single instance at a time, owned by `GatewayConnectionManager`. Created via `GatewayClientFactory.Create()`. Old instance disposed before creating new one. `OperatorClientChanged` event notifies consumers of swaps. diff --git a/src/OpenClaw.Connection/ConnectionStateMachine.cs b/src/OpenClaw.Connection/ConnectionStateMachine.cs index ff9f53fbb..83b9ff129 100644 --- a/src/OpenClaw.Connection/ConnectionStateMachine.cs +++ b/src/OpenClaw.Connection/ConnectionStateMachine.cs @@ -12,6 +12,7 @@ internal sealed class ConnectionStateMachine private RoleConnectionState _operatorState = RoleConnectionState.Idle; private RoleConnectionState _nodeState = RoleConnectionState.Idle; private string? _operatorError; + private OpenClaw.Shared.GatewayErrorKind? _operatorErrorKind; private string? _nodeError; private string? _operatorCredentialSource; private string? _nodeCredentialSource; @@ -154,6 +155,7 @@ public void Reset() _operatorState = RoleConnectionState.Idle; _nodeState = _nodeEnabled ? RoleConnectionState.Idle : RoleConnectionState.Disabled; _operatorError = null; + _operatorErrorKind = null; _nodeError = null; _operatorCredentialSource = null; _nodeCredentialSource = null; @@ -207,6 +209,12 @@ internal void SetOperatorCredentialResolution(GatewayCredentialResolution resolu RebuildSnapshot(); } + internal void SetOperatorErrorKind(OpenClaw.Shared.GatewayErrorKind? kind) + { + _operatorErrorKind = kind; + RebuildSnapshot(); + } + /// Update node info (device ID, pairing status, optional request ID) in the snapshot. internal void SetNodeInfo( string? deviceId, @@ -293,6 +301,7 @@ private void ApplyTransition(ConnectionTrigger trigger, string? detail) case ConnectionTrigger.ConnectRequested: _operatorState = RoleConnectionState.Connecting; _operatorError = null; + _operatorErrorKind = null; break; case ConnectionTrigger.ConnectRequestSent: @@ -304,6 +313,7 @@ private void ApplyTransition(ConnectionTrigger trigger, string? detail) case ConnectionTrigger.HandshakeSucceeded: _operatorState = RoleConnectionState.Connected; _operatorError = null; + _operatorErrorKind = null; break; case ConnectionTrigger.PairingPending: @@ -317,16 +327,19 @@ private void ApplyTransition(ConnectionTrigger trigger, string? detail) case ConnectionTrigger.PairingRejected: _operatorState = RoleConnectionState.Error; _operatorError = detail ?? "Pairing rejected"; + _operatorErrorKind = OpenClaw.Shared.GatewayErrorKind.PairingRejected; break; case ConnectionTrigger.AuthenticationFailed: _operatorState = RoleConnectionState.Error; _operatorError = detail ?? "Authentication failed"; + _operatorErrorKind ??= OpenClaw.Shared.GatewayErrorClassifier.ClassifyWithCode(_operatorError); break; case ConnectionTrigger.RateLimited: _operatorState = RoleConnectionState.Error; _operatorError = detail ?? "Rate limited"; + _operatorErrorKind = OpenClaw.Shared.GatewayErrorKind.RateLimited; break; case ConnectionTrigger.WebSocketDisconnected: @@ -339,12 +352,14 @@ private void ApplyTransition(ConnectionTrigger trigger, string? detail) { _operatorState = RoleConnectionState.Connecting; _operatorError = null; + _operatorErrorKind = null; } break; case ConnectionTrigger.WebSocketError: _operatorState = RoleConnectionState.Error; _operatorError = detail ?? "WebSocket error"; + _operatorErrorKind ??= OpenClaw.Shared.GatewayErrorClassifier.Classify(_operatorError); break; case ConnectionTrigger.DisconnectRequested: @@ -352,6 +367,7 @@ private void ApplyTransition(ConnectionTrigger trigger, string? detail) _operatorState = RoleConnectionState.Idle; _nodeState = _nodeEnabled ? RoleConnectionState.Idle : RoleConnectionState.Disabled; _operatorError = null; + _operatorErrorKind = null; _nodeError = null; _operatorCredentialSource = null; _nodeCredentialSource = null; @@ -368,6 +384,7 @@ private void ApplyTransition(ConnectionTrigger trigger, string? detail) case ConnectionTrigger.ReconnectScheduled: _operatorState = RoleConnectionState.Connecting; _operatorError = null; + _operatorErrorKind = null; break; case ConnectionTrigger.ReconnectSuppressed: @@ -377,6 +394,7 @@ private void ApplyTransition(ConnectionTrigger trigger, string? detail) case ConnectionTrigger.Cancelled: _operatorState = RoleConnectionState.Idle; _operatorError = null; + _operatorErrorKind = null; break; // ─── Node transitions ─── @@ -424,6 +442,7 @@ private void RebuildSnapshot() OverallState = GatewayConnectionSnapshot.DeriveOverall(_operatorState, _nodeState, _nodeEnabled), OperatorState = _operatorState, OperatorError = _operatorError, + OperatorErrorKind = _operatorErrorKind, OperatorCredentialSource = _operatorCredentialSource, OperatorCredentialStatus = _operatorCredentialStatus, OperatorCredentialFallbackUsed = _operatorCredentialFallbackUsed, diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index 3e7c4c57a..21f153fb9 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -57,12 +57,22 @@ public sealed class GatewayConnectionManager : IGatewayConnectionManager private readonly IClock _clock; private readonly Func? _shouldStartNodeConnection; private readonly Func _reconnectDelay; + private readonly Func>? + _endpointProvenanceProbe; private readonly SemaphoreSlim _transitionSemaphore = new(1, 1); private readonly SemaphoreSlim _nodeStartSemaphore = new(1, 1); private readonly object _nodeOperationLock = new(); private readonly object _devicePairReconnectLock = new(); private readonly object _disposeLock = new(); private readonly object _telemetryLock = new(); + private readonly object _operatorFailureLock = new(); + private readonly object _connectionIntentLock = new(); + private readonly HashSet _userDisconnectedGatewayIds = new(StringComparer.Ordinal); + // Shared exclusive lease serializing destructive gateway lifecycle operations (manual WSL + // start/stop/restart vs auto-repair distro restart). _manualLeaseHolders counts manual holders so + // the monitor can additionally suppress starting new repairs while a manual action runs. + private readonly SemaphoreSlim _gatewayLifecycleLease = new(1, 1); + private int _manualLeaseHolders; private long _generation; private CancellationTokenSource? _operationCts; @@ -76,6 +86,7 @@ public sealed class GatewayConnectionManager : IGatewayConnectionManager private Task? _disposeTask; private bool _gatewayNeedsV2Signature; // remembered across reconnects private string? _operatorTokenRecoveryAttemptedGatewayId; + private string? _nodeTokenRecoveryAttemptedGatewayId; private string? _lastAutoApprovedDevicePairRequestId; // prevent role-upgrade auto-approve loops private string? _devicePairAutoApproveInFlight; // atomic guard against concurrent approval of same requestId private bool _devicePairReconnectInFlight; @@ -89,6 +100,8 @@ public sealed class GatewayConnectionManager : IGatewayConnectionManager private TelemetryAttempt? _operatorTelemetryAttempt; private TelemetryAttempt? _nodeTelemetryAttempt; private GatewayConnectionSnapshot _lastTelemetrySnapshot = GatewayConnectionSnapshot.Idle; + private long _pendingOperatorFailureGeneration; + private GatewayErrorKind? _pendingOperatorFailureKind; private const string MissingNodeCredentialMessage = "No node credential available. Re-pair this PC or add a shared/bootstrap gateway token."; @@ -117,7 +130,9 @@ public GatewayConnectionManager( ConnectionDiagnostics? diagnostics = null, ISshTunnelManager? tunnelManager = null, Func? shouldStartNodeConnection = null, - Func? reconnectDelay = null) + Func? reconnectDelay = null, + Func>? + endpointProvenanceProbe = null) { _credentialResolver = credentialResolver ?? throw new ArgumentNullException(nameof(credentialResolver)); _clientFactory = clientFactory ?? throw new ArgumentNullException(nameof(clientFactory)); @@ -130,6 +145,7 @@ public GatewayConnectionManager( _clock = clock ?? SystemClock.Instance; _shouldStartNodeConnection = shouldStartNodeConnection; _reconnectDelay = reconnectDelay ?? Task.Delay; + _endpointProvenanceProbe = endpointProvenanceProbe; _diagnostics = diagnostics ?? new ConnectionDiagnostics(clock: clock); _diagnostics.EventRecorded += (_, e) => DiagnosticEvent?.Invoke(this, e); @@ -163,6 +179,9 @@ public async Task ConnectAsync(string? gatewayId = null) await _transitionSemaphore.WaitAsync(); try { + var targetId = gatewayId ?? _registry.ActiveGatewayId; + if (targetId is not null) + SetGatewayConnectionIntent(targetId, shouldBeConnected: true); await ConnectCoreAsync(gatewayId, "connect"); } finally @@ -286,6 +305,37 @@ private async Task ConnectCoreAsync(string? gatewayId = null, string operation = return; } + var endpointAuthorization = await AuthorizeCredentialForEndpointAsync( + record, + credential, + _operationCts!.Token).ConfigureAwait(false); + if (_disposed || + Interlocked.Read(ref _generation) != gen || + _operationCts?.IsCancellationRequested != false) + { + return; + } + if (!endpointAuthorization.Allowed) + { + _stateMachine.TryTransition(ConnectionTrigger.ConnectRequested); + _stateMachine.SetOperatorCredentialResolution(credentialResolution); + _stateMachine.SetOperatorErrorKind(endpointAuthorization.FailureKind); + _stateMachine.TryTransition( + endpointAuthorization.FailureKind == GatewayErrorKind.Network + ? ConnectionTrigger.WebSocketError + : ConnectionTrigger.AuthenticationFailed, + endpointAuthorization.Detail); + _diagnostics.Record("setup", "Blocked strong credential before managed-local endpoint ownership was proven", endpointAuthorization.Detail); + CompleteOperatorTelemetryAttempt( + gen, + "failure", + endpointAuthorization.FailureKind == GatewayErrorKind.Network + ? ConnectionErrorCategory.NetworkUnreachable + : ConnectionErrorCategory.AuthFailure); + EmitStateChanged(); + return; + } + // Transition to Connecting var prevState = _stateMachine.Current.OverallState; _stateMachine.TryTransition(ConnectionTrigger.ConnectRequested); @@ -339,6 +389,25 @@ tunnel.SshPort is < 1 or > 65535 || } var diagLogger = new DiagnosticTeeLogger(_logger, _diagnostics); var lifecycle = _clientFactory.Create(connectUrl, credential, perGatewayIdentityDir, diagLogger); + lifecycle.DataClient.ReconnectAuthorizationAsync = async cancellationToken => + { + if (!IsCurrentGatewayAttempt(gen, record.Id) || + !IsAutomaticReconnectAllowed(record.Id)) + { + return new ReconnectAuthorizationResult( + false, + GatewayErrorKind.Unknown, + "Connection attempt was superseded or explicitly disconnected."); + } + var authorization = await AuthorizeCredentialForEndpointAsync( + record, + credential, + cancellationToken).ConfigureAwait(false); + return new ReconnectAuthorizationResult( + authorization.Allowed, + authorization.FailureKind, + authorization.Detail); + }; _activeLifecycle = lifecycle; OperatorClientChanged?.Invoke(this, new OperatorClientChangedEventArgs { @@ -358,6 +427,11 @@ tunnel.SshPort is < 1 or > 65535 || if (!IsCurrentGatewayAttempt(gen, subscribedGatewayId)) return; _ = HandleAuthenticationFailedAsync(msg, gen); }; + lifecycle.DataClient.ConnectionFailure += (s, kind) => + { + if (!IsCurrentGatewayAttempt(gen, subscribedGatewayId)) return; + RecordOperatorFailureKind(gen, kind); + }; lifecycle.DataClient.TransportConnected += (s, e) => { if (!IsCurrentGatewayAttempt(gen, subscribedGatewayId)) return; @@ -490,6 +564,26 @@ tunnel.SshPort is < 1 or > 65535 || return null; } + var nodeEndpointAuthorization = await AuthorizeCredentialForEndpointAsync( + record, + nodeCredential, + _operationCts?.Token ?? CancellationToken.None).ConfigureAwait(false); + if (_disposed || + Interlocked.Read(ref _generation) != gen || + _operationCts?.IsCancellationRequested != false) + { + return null; + } + if (!nodeEndpointAuthorization.Allowed) + { + _diagnostics.Record("setup", "Blocked node credential before managed-local endpoint ownership was proven", nodeEndpointAuthorization.Detail); + _stateMachine.SetNodeCredentialResolution(nodeCredentialResolution); + _stateMachine.BlockNodeStart(nodeEndpointAuthorization.Detail, preserveCredentialResolution: true); + EmitStateChanged(); + RecordNodePreflightTelemetryFailure(ConnectionErrorCategory.AuthFailure); + return null; + } + _diagnostics.RecordCredentialResolutionResult(nodeCredentialResolution); if (!preservesOperatorConnection) _stateMachine.SetOperatorCredentialSource(null); @@ -558,6 +652,23 @@ public async Task DisconnectAsync() } } + public async Task DisconnectByUserAsync() + { + ThrowIfDisposed(); + await _transitionSemaphore.WaitAsync(); + try + { + var gatewayId = _registry.ActiveGatewayId; + if (gatewayId is not null) + SetGatewayConnectionIntent(gatewayId, shouldBeConnected: false); + await DisconnectCoreAsync(); + } + finally + { + _transitionSemaphore.Release(); + } + } + /// Core disconnect logic. Caller must hold . private async Task DisconnectCoreAsync() { @@ -582,6 +693,9 @@ public async Task ReconnectAsync() await _transitionSemaphore.WaitAsync(); try { + var gatewayId = _registry.ActiveGatewayId; + if (gatewayId is not null) + SetGatewayConnectionIntent(gatewayId, shouldBeConnected: true); await DisconnectCoreAsync(); await ConnectCoreAsync(operation: "reconnect"); } @@ -591,6 +705,132 @@ public async Task ReconnectAsync() } } + /// + /// Reconnects the active gateway ONLY if is still the active + /// gateway, and honors cancellation. Used by managed-local auto-repair so a gateway switch + /// during a repair cannot disrupt the newly selected gateway, and so a shutdown-cancelled + /// repair does not drive a reconnect into a disposing manager. Returns true if it reconnected, + /// false if the active gateway changed (no-op). + /// + public async Task ReconnectIfCurrentAsync(string gatewayId, CancellationToken cancellationToken = default) + { + ThrowIfDisposed(); + await _transitionSemaphore.WaitAsync(cancellationToken); + try + { + if (!IsAutomaticReconnectAllowed(gatewayId)) + return false; + + if (!string.Equals(_registry.ActiveGatewayId, gatewayId, StringComparison.Ordinal)) + return false; + + cancellationToken.ThrowIfCancellationRequested(); + await DisconnectCoreAsync(); + cancellationToken.ThrowIfCancellationRequested(); + + // Re-validate under the same semaphore hold before connecting: an out-of-band SetActive + // (e.g. a UI gateway switch that mutates the registry outside this manager) could have + // changed the active gateway while we were disconnecting. Fail closed if so. + if (!string.Equals(_registry.ActiveGatewayId, gatewayId, StringComparison.Ordinal)) + return false; + if (!IsAutomaticReconnectAllowed(gatewayId)) + return false; + + // Connect the PINNED gateway id, not "whatever is active now" — ConnectCoreAsync otherwise + // re-reads ActiveGatewayId, which the UI can mutate outside this semaphore, so an + // unpinned connect could bring up a different gateway than the one this repair targeted. + await ConnectCoreAsync(gatewayId, operation: "reconnect"); + + // Report whether a connection was actually LAUNCHED for the pinned gateway. ConnectCoreAsync + // bails to the Error state (without creating a client) when the record was removed mid-flight + // or credential resolution failed — returning true there would let auto-repair treat a + // credential failure as "reconnected, just unverified" and restart WSL, which cannot fix + // credentials. Require a non-Error operator state AND the pinned record still active. + return _stateMachine.Current.OperatorState is not RoleConnectionState.Error + && _registry.GetById(gatewayId) is not null + && string.Equals(_registry.ActiveGatewayId, gatewayId, StringComparison.Ordinal); + } + finally + { + _transitionSemaphore.Release(); + } + } + + public void SetGatewayConnectionIntent(string gatewayId, bool shouldBeConnected) + { + if (string.IsNullOrWhiteSpace(gatewayId)) + return; + + lock (_connectionIntentLock) + { + if (shouldBeConnected) + _userDisconnectedGatewayIds.Remove(gatewayId); + else + _userDisconnectedGatewayIds.Add(gatewayId); + } + } + + public bool IsAutomaticReconnectAllowed(string gatewayId) + { + if (string.IsNullOrWhiteSpace(gatewayId)) + return false; + lock (_connectionIntentLock) + { + return !_userDisconnectedGatewayIds.Contains(gatewayId); + } + } + + /// + /// True while a user-initiated gateway lifecycle action (manual WSL start/stop/restart) is in + /// progress. Managed-local auto-repair observes this to suppress STARTING a new repair. + /// + public bool IsManualGatewayLifecycleInProgress => Volatile.Read(ref _manualLeaseHolders) > 0; + + /// + /// Acquires the shared gateway-lifecycle lease for a user-initiated manual WSL operation, awaiting + /// it so the manual op is MUTUALLY EXCLUSIVE with an in-flight auto-repair distro restart (whose + /// host-side terminate could otherwise kill the manual op's freshly booted VM). Also marks a manual + /// holder so the monitor additionally suppresses starting new repairs. Dispose releases the lease. + /// + public async Task BeginManualGatewayLifecycleOperationAsync(CancellationToken cancellationToken = default) + { + await _gatewayLifecycleLease.WaitAsync(cancellationToken).ConfigureAwait(false); + Interlocked.Increment(ref _manualLeaseHolders); + return new LeaseScope(this, isManual: true); + } + + /// + /// Non-blocking attempt to acquire the shared gateway-lifecycle lease for an automatic repair's + /// destructive restart. Returns null if a manual (or another) operation holds it, so the coordinator + /// aborts instead of running a concurrent restart. Dispose releases the lease. + /// + public IDisposable? TryAcquireGatewayLifecycleLease() + => _gatewayLifecycleLease.Wait(0) ? new LeaseScope(this, isManual: false) : null; + + private void ReleaseGatewayLifecycleLease(bool isManual) + { + if (isManual) + Interlocked.Decrement(ref _manualLeaseHolders); + + // Guard against a shutdown dispose-race: the manager may dispose the lease while a manual op + // still holds a scope, so releasing here can hit a disposed semaphore. The manual-holder count + // is already decremented above, so the monitor cannot get stuck-suppressed. + // slopwatch-ignore: SW003 Shutdown dispose-race is expected; the count is already corrected and no caller state improves by surfacing it. + try { _gatewayLifecycleLease.Release(); } + catch (ObjectDisposedException) { } + catch (SemaphoreFullException) { } + } + + private sealed class LeaseScope(GatewayConnectionManager owner, bool isManual) : IDisposable + { + private int _disposed; + public void Dispose() + { + if (Interlocked.Exchange(ref _disposed, 1) == 0) + owner.ReleaseGatewayLifecycleLease(isManual); + } + } + public async Task SwitchGatewayAsync(string gatewayId) { ThrowIfDisposed(); @@ -606,6 +846,7 @@ public async Task SwitchGatewayAsync(string gatewayId) var previousActiveId = _registry.ActiveGatewayId; _diagnostics.Record("state", $"Switching active gateway to {gatewayId}"); + SetGatewayConnectionIntent(gatewayId, shouldBeConnected: true); _registry.SetActive(gatewayId); try { @@ -691,6 +932,7 @@ public async Task ApplySetupCodeAsync(string setupCode, SshTunn _logger.Warn($"[ConnMgr] Failed to persist setup-code gateway update: {ex.Message}"); return new SetupCodeResult(SetupCodeOutcome.ConnectionFailed, ex.Message); } + SetGatewayConnectionIntent(recordId, shouldBeConnected: true); // 3. Disconnect current gateway only after the new active gateway is persisted. await DisconnectCoreAsync(); @@ -740,6 +982,27 @@ public async Task ConnectWithSharedTokenAsync( if (existing != null && hasDurableTokens) { + var validationRecord = existing with + { + Url = gatewayUrl, + SharedGatewayToken = token, + SshTunnel = sshTunnel, + }; + var validationCredential = new GatewayCredential( + token, + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken); + var validationAuthorization = await AuthorizeCredentialForEndpointAsync( + validationRecord, + validationCredential, + CancellationToken.None).ConfigureAwait(false); + if (!validationAuthorization.Allowed) + { + return new SetupCodeResult( + SetupCodeOutcome.ConnectionFailed, + validationAuthorization.Detail); + } + var validation = await ValidateSharedTokenBeforeReplacementAsync( gatewayUrl, token, @@ -774,6 +1037,7 @@ public async Task ConnectWithSharedTokenAsync( _logger.Warn($"[ConnMgr] Failed to persist shared-token gateway update: {ex.Message}"); return new SetupCodeResult(SetupCodeOutcome.ConnectionFailed, ex.Message); } + SetGatewayConnectionIntent(recordId, shouldBeConnected: true); // Disconnect current gateway only after replacement credentials have been validated and persisted. await DisconnectCoreAsync(); @@ -818,6 +1082,14 @@ private async Task ValidateSharedTokenBeforeReplacementAsync( { UseV2Signature = existing.IsLocal || existing.RequiresV2Signature }; + // This is a one-shot validation client. A reconnect would reuse the strong shared token after + // ownership may have changed; fail the validation instead and let the caller retry from a new + // provenance preflight. + client.ReconnectAuthorizationAsync = _ => Task.FromResult( + new ReconnectAuthorizationResult( + false, + GatewayErrorKind.Auth, + "Shared-token validation is one-shot.")); var completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); client.HandshakeSucceeded += (_, _) => @@ -826,7 +1098,7 @@ private async Task ValidateSharedTokenBeforeReplacementAsync( completion.TrySetResult(new SetupCodeResult(SetupCodeOutcome.ConnectionFailed, message)); client.StatusChanged += (_, status) => { - if (status == ConnectionStatus.Error) + if (status is ConnectionStatus.Error or ConnectionStatus.Disconnected) completion.TrySetResult(new SetupCodeResult(SetupCodeOutcome.ConnectionFailed, "Shared token validation failed")); }; @@ -869,6 +1141,7 @@ private async Task HandleOperatorStatusChangedAsync(ConnectionStatus status, lon { case ConnectionStatus.Connected: _diagnostics.RecordWebSocketEvent("WebSocket connected"); + ClearOperatorFailureKind(gen); _stateMachine.TryTransition(ConnectionTrigger.WebSocketConnected); break; case ConnectionStatus.Disconnected: @@ -884,7 +1157,18 @@ private async Task HandleOperatorStatusChangedAsync(ConnectionStatus status, lon case ConnectionStatus.Error: _diagnostics.RecordWebSocketEvent("WebSocket error"); if (_stateMachine.Current.OperatorState != RoleConnectionState.PairingRequired) + { + // AuthenticationFailed and Status=Error are raised back-to-back and handled + // asynchronously. If the auth handler already promoted the failure to a more + // specific terminal kind (for example LocalPortConflict), never let the later + // generic status handler overwrite it with the original token/transport kind. + if (_stateMachine.Current.OperatorState != RoleConnectionState.Error || + _stateMachine.Current.OperatorErrorKind is null) + { + _stateMachine.SetOperatorErrorKind(ReadOperatorFailureKind(gen)); + } _stateMachine.TryTransition(ConnectionTrigger.WebSocketError, "Transport error"); + } CompleteOperatorTelemetryAttempt( gen, "failure", @@ -909,10 +1193,42 @@ private async Task HandleAuthenticationFailedAsync(string message, long gen) { if (Interlocked.Read(ref _generation) != gen) return; - if (TryScheduleOperatorTokenRecovery(message, gen)) + var failureKind = + ReadOperatorFailureKind(gen) ?? GatewayErrorClassifier.ClassifyWithCode(message); + var activeRecord = _activeGatewayRecordId is null + ? null + : _registry.GetById(_activeGatewayRecordId); + var provenance = activeRecord is not null && + GatewayRecordEditing.ResolveManagedDistroName(activeRecord) is not null && + _endpointProvenanceProbe is not null + ? await _endpointProvenanceProbe(activeRecord, CancellationToken.None).ConfigureAwait(false) + : null; + var unexpectedManagedLocalOwner = + provenance?.Kind is GatewayEndpointProvenanceKind.ConflictingOpenClawGateway + or GatewayEndpointProvenanceKind.UnknownListener; + + // A wrong local process may report either shared-token mismatch OR device-token mismatch. + // In both cases the real failure is endpoint identity, not credentials: never disclose the + // shared/bootstrap fallback and let the provenance-gated collision repair own recovery. + if (activeRecord is not null && + unexpectedManagedLocalOwner && + failureKind is GatewayErrorKind.DeviceTokenMismatch or GatewayErrorKind.Auth) + { + failureKind = GatewayErrorKind.LocalPortConflict; + _diagnostics.Record( + "setup", + "Managed local gateway port is owned by a different or unverified process", + $"gatewayId={activeRecord.Id}"); + } + + if (failureKind == GatewayErrorKind.DeviceTokenMismatch && + await TryScheduleOperatorTokenRecoveryAsync(message, gen).ConfigureAwait(false)) + { return; + } _diagnostics.Record("error", "Authentication failed", message); + _stateMachine.SetOperatorErrorKind(failureKind); _stateMachine.TryTransition(ConnectionTrigger.AuthenticationFailed, message); CompleteOperatorTelemetryAttempt( gen, @@ -926,7 +1242,36 @@ private async Task HandleAuthenticationFailedAsync(string message, long gen) } } - private bool TryScheduleOperatorTokenRecovery(string message, long gen) + private void RecordOperatorFailureKind(long generation, GatewayErrorKind kind) + { + lock (_operatorFailureLock) + { + _pendingOperatorFailureGeneration = generation; + _pendingOperatorFailureKind = kind; + } + } + + private GatewayErrorKind? ReadOperatorFailureKind(long generation) + { + lock (_operatorFailureLock) + { + return _pendingOperatorFailureGeneration == generation + ? _pendingOperatorFailureKind + : null; + } + } + + private void ClearOperatorFailureKind(long generation) + { + lock (_operatorFailureLock) + { + if (_pendingOperatorFailureGeneration != generation) + return; + _pendingOperatorFailureKind = null; + } + } + + private async Task TryScheduleOperatorTokenRecoveryAsync(string message, long gen) { if (!IsOperatorDeviceTokenMismatch(message) || _activeGatewayRecordId == null || @@ -937,23 +1282,231 @@ private bool TryScheduleOperatorTokenRecovery(string message, long gen) } var record = _registry.GetById(_activeGatewayRecordId); - if (record == null || string.IsNullOrWhiteSpace(record.BootstrapToken)) + if (record == null) + return false; + + // Recovery clears the rejected device token and reconnects, letting CredentialResolver + // fall back to the shared gateway token (preferred) or bootstrap token. Setup clears the + // bootstrap token once pairing is durable but leaves the shared token, so a later stale + // device token can still self-recover without asking the user for a new token. With no + // fallback at all, clearing would only loop, so leave the gateway in Error for manual re-pair. + var hasSharedToken = !string.IsNullOrWhiteSpace(record.SharedGatewayToken); + var hasBootstrapToken = !string.IsNullOrWhiteSpace(record.BootstrapToken); + if (!hasSharedToken && !hasBootstrapToken) return false; + // SECURITY: clearing the device token downgrades to the more powerful shared/bootstrap + // credential. Only do this over a trusted endpoint so a hostile cleartext endpoint cannot + // return a device-token-mismatch to induce disclosure of the shared credential. + if (!await IsRecoverySafeEndpointAsync(record, CancellationToken.None).ConfigureAwait(false)) + { + _diagnostics.Record("credential", "Skipped operator token recovery: endpoint not trusted for credential fallback"); + return false; + } + if (!DeviceIdentity.TryClearDeviceToken(_activeIdentityPath, _logger)) return false; _operatorTokenRecoveryAttemptedGatewayId = _activeGatewayRecordId; - _diagnostics.Record("credential", "Cleared stale operator device token; reconnecting with bootstrap token"); + var fallbackLabel = hasSharedToken ? "shared gateway token" : "bootstrap token"; + _diagnostics.Record("credential", $"Cleared stale operator device token; reconnecting with {fallbackLabel}"); - ScheduleDelayedReconnect(gen, "[ConnMgr] Operator token recovery reconnect failed"); + ScheduleDelayedReconnect(gen, "[ConnMgr] Operator token recovery reconnect failed", gatewayId: record.Id); return true; } private static bool IsOperatorDeviceTokenMismatch(string message) => - message.Contains("device token mismatch", StringComparison.OrdinalIgnoreCase) || - message.Contains("AUTH_DEVICE_TOKEN_MISMATCH", StringComparison.OrdinalIgnoreCase); + GatewayErrorClassifier.ClassifyWithCode(message) == GatewayErrorKind.DeviceTokenMismatch; + + // Auto credential recovery clears a device token and falls back to a stronger shared/bootstrap + // credential. Restrict that to trusted endpoints (mirrors the Mac app, which only retries + // credentials on loopback or explicitly trusted transport): a loopback/local endpoint (traffic + // never leaves the machine), an owned SSH tunnel (encrypted, user-configured), or a validated + // TLS endpoint (wss/https). A plain ws:// remote endpoint is never eligible. + private async Task IsRecoverySafeEndpointAsync( + GatewayRecord record, + CancellationToken cancellationToken) + { + if (GatewayRecordEditing.IsLoopbackEndpoint(record.Url)) + { + if (record.IsLocal || GatewayRecordEditing.ResolveManagedDistroName(record) is not null) + { + if (_endpointProvenanceProbe is null) + return false; + return (await _endpointProvenanceProbe(record, cancellationToken).ConfigureAwait(false)).Kind == + GatewayEndpointProvenanceKind.ExpectedManagedGateway; + } + return true; + } + if (record.SshTunnel is not null) + return true; + if (string.IsNullOrWhiteSpace(record.Url)) + return false; + return Uri.TryCreate(record.Url, UriKind.Absolute, out var uri) && + (string.Equals(uri.Scheme, "wss", StringComparison.OrdinalIgnoreCase) || + string.Equals(uri.Scheme, "https", StringComparison.OrdinalIgnoreCase)); + } + + private async Task AuthorizeCredentialForEndpointAsync( + GatewayRecord record, + GatewayCredential credential, + CancellationToken cancellationToken) + { + var isStrongCredential = + credential.IsBootstrapToken || + string.Equals( + credential.Source, + CredentialResolver.SourceSharedGatewayToken, + StringComparison.Ordinal) || + string.Equals( + credential.Source, + CredentialResolver.SourceBootstrapToken, + StringComparison.Ordinal); + var isManagedLoopback = + record.SshTunnel is null && + (record.IsLocal || GatewayRecordEditing.ResolveManagedDistroName(record) is not null) && + GatewayRecordEditing.IsLoopbackEndpoint(record.Url); + if (!isManagedLoopback) + return EndpointCredentialAuthorization.AllowedResult; + if (!isStrongCredential) + { + // Still populate the shared provenance cache used by Chat/Dashboard, but a device token + // does not need the stronger-credential gate. + if (_endpointProvenanceProbe is not null) + _ = await _endpointProvenanceProbe(record, cancellationToken).ConfigureAwait(false); + return EndpointCredentialAuthorization.AllowedResult; + } + if (_endpointProvenanceProbe is null) + { + return new EndpointCredentialAuthorization( + false, + GatewayErrorKind.LocalPortConflict, + "Managed-local endpoint ownership could not be verified, so OpenClaw did not send the shared or bootstrap token."); + } + + var provenance = await _endpointProvenanceProbe(record, cancellationToken).ConfigureAwait(false); + if (provenance.Kind == GatewayEndpointProvenanceKind.ExpectedManagedGateway) + return EndpointCredentialAuthorization.AllowedResult; + + if (provenance.Kind == GatewayEndpointProvenanceKind.NoListener) + { + return new EndpointCredentialAuthorization( + false, + GatewayErrorKind.Network, + "The managed WSL gateway is not listening yet. Automatic repair can restart it without sending credentials."); + } + + return new EndpointCredentialAuthorization( + false, + GatewayErrorKind.LocalPortConflict, + provenance.Detail ?? + "The managed gateway address is owned by an unverified process. OpenClaw did not send the shared or bootstrap token."); + } + + private readonly record struct EndpointCredentialAuthorization( + bool Allowed, + GatewayErrorKind FailureKind, + string Detail) + { + public static EndpointCredentialAuthorization AllowedResult { get; } = + new(true, GatewayErrorKind.Unknown, string.Empty); + } + + // Node device-token recovery mirrors operator recovery but clears ONLY the node token and + // reconnects the node, leaving operator credentials untouched. Queued off the connection-failure + // handler (which runs under the connector's client-lifecycle lock) so the disk clear + reconnect + // never run under that lock. Generations captured at failure time are pinned so a newer node + // attempt supersedes this recovery. + private async Task HandleNodeDeviceTokenMismatchAsync(long lifecycleGeneration, long nodeGeneration) + { + try + { + if (!IsCurrentNodeAttempt(lifecycleGeneration, nodeGeneration)) + return; + + await _transitionSemaphore.WaitAsync(); + try + { + if (!IsCurrentNodeAttempt(lifecycleGeneration, nodeGeneration)) + return; + + var gatewayRecordId = _activeGatewayRecordId; + var identityPath = _activeIdentityPath; + if (gatewayRecordId == null || + identityPath == null || + _nodeTokenRecoveryAttemptedGatewayId == gatewayRecordId) + { + return; + } + + // The node reconnect needs a live operator; if the operator is down the operator + // recovery path drives the full reconnect instead. + if (_stateMachine.Current.OperatorState != RoleConnectionState.Connected) + return; + + var record = _registry.GetById(gatewayRecordId); + if (record == null) + return; + + var hasSharedToken = !string.IsNullOrWhiteSpace(record.SharedGatewayToken); + var hasBootstrapToken = !string.IsNullOrWhiteSpace(record.BootstrapToken); + if (!hasSharedToken && !hasBootstrapToken) + return; + + if (!await IsRecoverySafeEndpointAsync(record, CancellationToken.None).ConfigureAwait(false)) + { + _diagnostics.Record("credential", "Skipped node token recovery: endpoint not trusted for credential fallback"); + return; + } + + if (!DeviceIdentity.TryClearDeviceTokenForRole(identityPath, "node", _logger)) + return; + + _nodeTokenRecoveryAttemptedGatewayId = gatewayRecordId; + var fallbackLabel = hasSharedToken ? "shared gateway token" : "bootstrap token"; + _diagnostics.Record("credential", $"Cleared stale node device token; reconnecting node with {fallbackLabel}"); + } + finally + { + _transitionSemaphore.Release(); + } + + ScheduleDelayedNodeReconnect(lifecycleGeneration, nodeGeneration); + } + // slopwatch-ignore: SW003 Shutdown/disposal is expected; caller preserves safe state. + catch (ObjectDisposedException) { } + catch (Exception ex) + { + _logger.Warn($"[ConnMgr] Node token recovery failed: {ex.Message}"); + _diagnostics.Record("credential", "Node token recovery failed", ex.Message); + } + } + + // Reconnects the node outside the transition semaphore, pinning BOTH the lifecycle and node + // generations so a newer node attempt started during the delay wins and this superseded + // recovery is dropped. + private void ScheduleDelayedNodeReconnect(long lifecycleGeneration, long nodeGeneration) + { + _ = Task.Run(async () => + { + try + { + await _reconnectDelay(TimeSpan.FromMilliseconds(200)); + if (_disposed || !IsCurrentNodeAttempt(lifecycleGeneration, nodeGeneration)) + return; + + await StartNodeConnectionAsync(lifecycleGeneration, nodeGeneration); + } + // slopwatch-ignore: SW003 Shutdown cancellation or disposal is expected; caller preserves safe state. + catch (ObjectDisposedException) { } + catch (Exception ex) + { + _logger.Warn($"[ConnMgr] Node token recovery reconnect failed: {ex.Message}"); + _diagnostics.Record("credential", "Node token recovery reconnect failed", ex.Message); + } + }); + } private async Task HandleHandshakeSucceededAsync(long gen) { @@ -1168,13 +1721,15 @@ private void TrySchedulePostBootstrapOperatorReconnect( ScheduleDelayedReconnect( gen, "[ConnMgr] Post-bootstrap operator reconnect failed", - ex => _diagnostics.Record("credential", "Post-bootstrap operator reconnect failed", ex.Message)); + ex => _diagnostics.Record("credential", "Post-bootstrap operator reconnect failed", ex.Message), + gatewayId: gatewayRecordId); } private void ScheduleDelayedReconnect( long generation, string warningPrefix, - Action? onFailure = null) + Action? onFailure = null, + string? gatewayId = null) { _ = Task.Run(async () => { @@ -1184,7 +1739,14 @@ private void ScheduleDelayedReconnect( if (_disposed || Interlocked.Read(ref _generation) != generation) return; - await ReconnectAsync(); + // Pin the reconnect to the gateway this recovery was scheduled for. The generation + // check above does not cover an out-of-band SetActive (the UI sets the active gateway + // before calling SwitchGatewayAsync, which is what bumps the generation), so a global + // ReconnectAsync() could reconnect/disrupt a different gateway the user just switched to. + if (gatewayId is not null) + await ReconnectIfCurrentAsync(gatewayId); + else + await ReconnectAsync(); } // slopwatch-ignore: SW003 Shutdown cancellation or disposal is expected and the caller already preserves the safe state. catch (ObjectDisposedException) { } @@ -1662,6 +2224,25 @@ private async Task StartNodeConnectionCoreAsync( return false; } + var nodeEndpointAuthorization = await AuthorizeCredentialForEndpointAsync( + record, + nodeCredential, + cancellationToken).ConfigureAwait(false); + if (!nodeEndpointAuthorization.Allowed) + { + _diagnostics.Record( + "setup", + "Blocked node credential before managed-local endpoint ownership was proven", + nodeEndpointAuthorization.Detail); + await BlockNodeStartAsync( + nodeEndpointAuthorization.Detail, + cancellationToken, + expectedLifecycleGeneration, + nodeGeneration); + CompleteNodeTelemetryAttempt(nodeGeneration, "failure", ConnectionErrorCategory.AuthFailure); + return false; + } + await _transitionSemaphore.WaitAsync(cancellationToken); try { @@ -1689,6 +2270,26 @@ private async Task StartNodeConnectionCoreAsync( _diagnostics.Record("node", $"Starting node connection to {nodeConnectUrl}", $"Credential source: {nodeCredential.Source}"); + if (_nodeConnector is INodeConnectorReconnectPolicy reconnectPolicy) + { + reconnectPolicy.ReconnectAuthorizationAsync = async reconnectCancellationToken => + { + if (!IsExpectedNodeStartCurrent(expectedLifecycleGeneration, nodeGeneration)) + return new ReconnectAuthorizationResult( + false, + GatewayErrorKind.Unknown, + "Node attempt was superseded."); + var authorization = await AuthorizeCredentialForEndpointAsync( + record, + nodeCredential, + reconnectCancellationToken).ConfigureAwait(false); + return new ReconnectAuthorizationResult( + authorization.Allowed, + authorization.FailureKind, + authorization.Detail); + }; + } + try { await _nodeConnector.ConnectAsync(nodeConnectUrl, nodeCredential, activeIdentityPath, @@ -1757,6 +2358,13 @@ private void OnNodeConnectionFailure(object? sender, GatewayErrorKind errorKind) nodeGeneration, "failure", MapNodeConnectionErrorCategory(errorKind)); + + // A stale node DEVICE token is the one node failure we can self-recover: clear only the + // node device token and reconnect with the still-valid shared/bootstrap credential. Queue + // it off this handler — it is invoked under the connector's client-lifecycle lock, which + // requires prompt, non-reentrant subscribers, so the disk clear + reconnect must not run here. + if (errorKind == GatewayErrorKind.DeviceTokenMismatch) + _ = Task.Run(() => HandleNodeDeviceTokenMismatchAsync(lifecycleGeneration, nodeGeneration)); } private void OnNodeDeviceTokenReceived(object? sender, DeviceTokenReceivedEventArgs e) @@ -1785,6 +2393,7 @@ private async Task OnNodeStatusChangedAsync(ConnectionStatus status) { case ConnectionStatus.Connected: _stateMachine.TryTransition(ConnectionTrigger.NodeConnected); + _nodeTokenRecoveryAttemptedGatewayId = null; break; case ConnectionStatus.Connecting: _stateMachine.StartNodeConnecting(); @@ -1877,6 +2486,7 @@ private async Task OnNodePairingStatusChangedAsync( { case PairingStatus.Paired: _stateMachine.TryTransition(ConnectionTrigger.NodePaired); + _nodeTokenRecoveryAttemptedGatewayId = null; Interlocked.Exchange(ref _lastAutoApprovedDevicePairRequestId, null); lock (_devicePairReconnectLock) { @@ -2546,6 +3156,7 @@ private static ConnectionErrorCategory MapNodeConnectionErrorCategory(GatewayErr { GatewayErrorKind.Auth or GatewayErrorKind.TokenDrift or + GatewayErrorKind.DeviceTokenMismatch or GatewayErrorKind.ScopeMismatch => ConnectionErrorCategory.AuthFailure, GatewayErrorKind.PairingRequired => ConnectionErrorCategory.PairingPending, GatewayErrorKind.PairingRejected => ConnectionErrorCategory.PairingRejected, @@ -2805,6 +3416,10 @@ private async Task DisposeCoreAsync() _transitionSemaphore.Dispose(); } + // slopwatch-ignore: SW003 Best-effort disposal of the lifecycle lease; failure cannot improve caller state. + try { _gatewayLifecycleLease.Dispose(); } + catch (Exception ex) { _logger.Debug($"[ConnMgr] Dispose: lifecycle lease dispose failed: {ex.Message}"); } + GC.SuppressFinalize(this); } } diff --git a/src/OpenClaw.Connection/GatewayConnectionSnapshot.cs b/src/OpenClaw.Connection/GatewayConnectionSnapshot.cs index f793bd618..4ba8ebd36 100644 --- a/src/OpenClaw.Connection/GatewayConnectionSnapshot.cs +++ b/src/OpenClaw.Connection/GatewayConnectionSnapshot.cs @@ -12,6 +12,7 @@ public sealed record GatewayConnectionSnapshot // ─── Operator ─── public RoleConnectionState OperatorState { get; init; } public string? OperatorError { get; init; } + public OpenClaw.Shared.GatewayErrorKind? OperatorErrorKind { get; init; } public bool OperatorPairingRequired { get; init; } public string? OperatorDeviceId { get; init; } public string? OperatorCredentialSource { get; init; } diff --git a/src/OpenClaw.Connection/GatewayEndpointProvenance.cs b/src/OpenClaw.Connection/GatewayEndpointProvenance.cs new file mode 100644 index 000000000..94dffd64d --- /dev/null +++ b/src/OpenClaw.Connection/GatewayEndpointProvenance.cs @@ -0,0 +1,34 @@ +namespace OpenClaw.Connection; + +/// Provenance of the process accepting connections for a gateway endpoint. +public enum GatewayEndpointProvenanceKind +{ + /// The probe does not apply (for example, a non-loopback remote endpoint). + NotApplicable, + + /// No listener currently owns the endpoint. + NoListener, + + /// The listener is the expected OS-owned WSL relay for this managed gateway. + ExpectedManagedGateway, + + /// A fully proven, obsolete native OpenClaw gateway owns the WSL gateway endpoint. + ConflictingOpenClawGateway, + + /// A listener exists, but its ownership cannot be proven safe. + UnknownListener, +} + +/// +/// Address-specific endpoint provenance. Process/task details are diagnostics only; no credential +/// material is ever included. +/// +public sealed record GatewayEndpointProvenance( + GatewayEndpointProvenanceKind Kind, + int Port, + int? ProcessId = null, + string? ProcessName = null, + DateTime? ProcessStartTimeUtc = null, + string? ProcessPath = null, + string? ScheduledTaskName = null, + string? Detail = null); diff --git a/src/OpenClaw.Connection/GatewayRecord.cs b/src/OpenClaw.Connection/GatewayRecord.cs index e1e9979a1..379bd3424 100644 --- a/src/OpenClaw.Connection/GatewayRecord.cs +++ b/src/OpenClaw.Connection/GatewayRecord.cs @@ -61,12 +61,139 @@ public static class GatewayRecordEditing /// Carries forward advanced per-gateway fields that the edit/connect forms don't expose, /// so editing name / token / URL / SSH settings can't silently drop them. A value already /// set on the rebuilt record wins (the form changed it); otherwise the existing record's - /// value is preserved. Currently scoped to . + /// value is preserved. Covers and — when the + /// gateway is still the same managed-local WSL gateway — the setup-managed ownership fields + /// (, , + /// ). Preserving those keeps a managed gateway's + /// keepalive and auto-repair working across an edit; dropping them silently disabled self-healing. + /// "Same gateway" means the endpoint URL is unchanged (a name/token-only edit) or differs only by + /// the standard localhost aliases localhost, 127.0.0.1, and ::1, with scheme, + /// port, path, and query unchanged. If the user repoints the URL or adds a tunnel, the record becomes + /// manual and all managed-ownership metadata is removed. /// public static GatewayRecord PreserveAdvancedFields(this GatewayRecord rebuilt, GatewayRecord? existing) - => existing is null - ? rebuilt - : rebuilt with { BrowserControlPort = rebuilt.BrowserControlPort ?? existing.BrowserControlPort }; + { + if (existing is null) + return rebuilt; + + var result = rebuilt with { BrowserControlPort = rebuilt.BrowserControlPort ?? existing.BrowserControlPort }; + + var stillSameManagedEndpoint = AreEquivalentManagedEndpoints(rebuilt.Url, existing.Url); + var existingManagedDistroName = ResolveManagedDistroName(existing); + var managedDistroName = + rebuilt.SetupManagedDistroName ?? + existingManagedDistroName; + + if (existing.IsLocal && + managedDistroName is not null && + rebuilt.SshTunnel is null && + stillSameManagedEndpoint) + { + result = result with + { + IsLocal = true, + // Migrate legacy "Local ()" ownership to the explicit durable marker. + SetupManagedDistroName = managedDistroName, + RequiresV2Signature = rebuilt.RequiresV2Signature || existing.RequiresV2Signature, + }; + } + else if (existingManagedDistroName is not null) + { + result = result with + { + IsLocal = OpenClaw.Shared.LocalGatewayUrlClassifier.IsLocalGatewayUrl(rebuilt.Url), + SetupManagedDistroName = null, + RequiresV2Signature = false, + FriendlyName = ParseLegacyManagedDistroName(result.FriendlyName) is not null + ? null + : result.FriendlyName, + }; + } + + return result; + } + + internal static bool AreEquivalentLoopbackEndpoints(string? left, string? right) + { + if (!Uri.TryCreate(left, UriKind.Absolute, out var leftUri) || + !Uri.TryCreate(right, UriKind.Absolute, out var rightUri) || + !leftUri.IsLoopback || + !rightUri.IsLoopback) + { + return false; + } + + return AreEquivalentManagedEndpoints(leftUri, rightUri); + } + + private static bool AreEquivalentManagedEndpoints(string? left, string? right) => + Uri.TryCreate(left, UriKind.Absolute, out var leftUri) && + Uri.TryCreate(right, UriKind.Absolute, out var rightUri) && + AreEquivalentManagedEndpoints(leftUri, rightUri); + + private static bool AreEquivalentManagedEndpoints(Uri leftUri, Uri rightUri) + { + var hostsEquivalent = + string.Equals( + NormalizeHost(leftUri.Host), + NormalizeHost(rightUri.Host), + StringComparison.OrdinalIgnoreCase) || + (leftUri.IsLoopback && + rightUri.IsLoopback && + IsStandardLoopbackAlias(leftUri.Host) && + IsStandardLoopbackAlias(rightUri.Host)); + + return hostsEquivalent && + string.Equals(leftUri.Scheme, rightUri.Scheme, StringComparison.OrdinalIgnoreCase) && + leftUri.Port == rightUri.Port && + string.Equals(leftUri.UserInfo, rightUri.UserInfo, StringComparison.Ordinal) && + string.Equals(leftUri.AbsolutePath, rightUri.AbsolutePath, StringComparison.Ordinal) && + string.Equals(leftUri.Query, rightUri.Query, StringComparison.Ordinal) && + string.Equals(leftUri.Fragment, rightUri.Fragment, StringComparison.Ordinal); + } + + private static bool IsStandardLoopbackAlias(string host) + { + var normalized = NormalizeHost(host); + return string.Equals(normalized, "localhost", StringComparison.OrdinalIgnoreCase) || + string.Equals(normalized, "127.0.0.1", StringComparison.Ordinal) || + string.Equals(normalized, "::1", StringComparison.Ordinal); + } + + private static string NormalizeHost(string host) => + host.Trim().TrimStart('[').TrimEnd(']').TrimEnd('.'); + + public static bool IsLoopbackEndpoint(string? url) => + Uri.TryCreate(url, UriKind.Absolute, out var uri) && uri.IsLoopback; + + public static string? ResolveManagedDistroName(GatewayRecord record) + { + if (!string.IsNullOrWhiteSpace(record.SetupManagedDistroName)) + return record.SetupManagedDistroName; + + if (!record.IsLocal || + !OpenClaw.Shared.LocalGatewayUrlClassifier.IsLocalGatewayUrl(record.Url) || + ParseLegacyManagedDistroName(record.FriendlyName) is not { } distro) + { + return null; + } + + return distro; + } + + private static string? ParseLegacyManagedDistroName(string? friendlyName) + { + const string prefix = "Local ("; + if (string.IsNullOrWhiteSpace(friendlyName) || + !friendlyName.StartsWith(prefix, StringComparison.Ordinal) || + !friendlyName.EndsWith(')')) + { + return null; + } + + var distro = friendlyName[prefix.Length..^1].Trim(); + return string.IsNullOrWhiteSpace(distro) ? null : distro; + } } /// Per-gateway SSH tunnel configuration. diff --git a/src/OpenClaw.Connection/GatewayRegistry.cs b/src/OpenClaw.Connection/GatewayRegistry.cs index 38e99caed..1274b32f2 100644 --- a/src/OpenClaw.Connection/GatewayRegistry.cs +++ b/src/OpenClaw.Connection/GatewayRegistry.cs @@ -354,11 +354,20 @@ private bool MigrateFromSettingsCore( } var id = Guid.NewGuid().ToString(); + var isLocal = LocalGatewayUrlClassifier.IsLocalGatewayUrl(gatewayUrl); + var setupManagedDistroName = + isLocal && !useSshTunnel + ? TryReadSetupManagedDistroName(settingsDir, gatewayUrl) + : null; var record = new GatewayRecord { Id = id, Url = gatewayUrl, - IsLocal = LocalGatewayUrlClassifier.IsLocalGatewayUrl(gatewayUrl), + IsLocal = isLocal, + FriendlyName = setupManagedDistroName is null + ? null + : $"Local ({setupManagedDistroName})", + SetupManagedDistroName = setupManagedDistroName, SharedGatewayToken = string.IsNullOrWhiteSpace(bootstrapToken) ? token : null, BootstrapToken = !string.IsNullOrWhiteSpace(bootstrapToken) ? bootstrapToken : null, SshTunnel = useSshTunnel @@ -400,6 +409,55 @@ private bool MigrateFromSettingsCore( return true; } + private static string? TryReadSetupManagedDistroName( + string settingsDir, + string gatewayUrl) + { + try + { + var direct = Environment.GetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR"); + var localRoot = Environment.GetEnvironmentVariable("OPENCLAW_TRAY_LOCALAPPDATA_DIR") + ?? Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData); + var settingsDirectoryName = Path.GetFileName( + settingsDir.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + var preferredDataDirectory = settingsDirectoryName.EndsWith( + "-Dev", + StringComparison.OrdinalIgnoreCase) + ? "OpenClawTray-Dev" + : "OpenClawTray"; + var alternateDataDirectory = preferredDataDirectory == "OpenClawTray" + ? "OpenClawTray-Dev" + : "OpenClawTray"; + var candidates = string.IsNullOrWhiteSpace(direct) + ? new[] + { + Path.Combine(localRoot, preferredDataDirectory, "setup-state.json"), + Path.Combine(localRoot, alternateDataDirectory, "setup-state.json"), + } + : [Path.Combine(direct, "setup-state.json")]; + foreach (var statePath in candidates) + { + if (!File.Exists(statePath)) + continue; + using var document = System.Text.Json.JsonDocument.Parse(File.ReadAllText(statePath)); + if (document.RootElement.TryGetProperty("DistroName", out var distroElement) && + !string.IsNullOrWhiteSpace(distroElement.GetString()) && + document.RootElement.TryGetProperty("GatewayUrl", out var gatewayUrlElement) && + GatewayRecordEditing.AreEquivalentLoopbackEndpoints( + gatewayUrl, + gatewayUrlElement.GetString())) + { + return distroElement.GetString(); + } + } + return null; + } + catch + { + return null; + } + } + private sealed class RegistryData { public List? Gateways { get; set; } diff --git a/src/OpenClaw.Connection/IGatewayConnectionManager.cs b/src/OpenClaw.Connection/IGatewayConnectionManager.cs index 3bbaaaf5a..194d26af7 100644 --- a/src/OpenClaw.Connection/IGatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/IGatewayConnectionManager.cs @@ -22,8 +22,25 @@ public interface IGatewayConnectionManager : IDisposable, IAsyncDisposable Task ConnectAsync(string? gatewayId = null); Task ConnectNodeOnlyAsync(string? gatewayId = null); Task DisconnectAsync(); + Task DisconnectByUserAsync(); Task ReconnectAsync(); + Task ReconnectIfCurrentAsync(string gatewayId, CancellationToken cancellationToken = default); Task SwitchGatewayAsync(string gatewayId); + void SetGatewayConnectionIntent(string gatewayId, bool shouldBeConnected); + bool IsAutomaticReconnectAllowed(string gatewayId); + + /// + /// True while a user-initiated gateway lifecycle action (manual WSL start/stop/restart) is in + /// progress. Managed-local auto-repair observes this to suppress itself so a manual restart and an + /// automatic repair cannot run concurrent distro restarts. + /// + bool IsManualGatewayLifecycleInProgress { get; } + + /// + /// Acquires the shared gateway-lifecycle lease for a manual WSL operation (awaiting it so it is + /// mutually exclusive with an in-flight auto-repair restart). Dispose the returned scope to release. + /// + Task BeginManualGatewayLifecycleOperationAsync(CancellationToken cancellationToken = default); /// /// Drive the node connection for the active gateway and await its terminal state. diff --git a/src/OpenClaw.Connection/INodeConnector.cs b/src/OpenClaw.Connection/INodeConnector.cs index 397f0f2a5..e263165e3 100644 --- a/src/OpenClaw.Connection/INodeConnector.cs +++ b/src/OpenClaw.Connection/INodeConnector.cs @@ -59,6 +59,11 @@ public interface INodeConnectorTelemetryEvents event EventHandler ConnectionFailure; } +public interface INodeConnectorReconnectPolicy +{ + Func>? ReconnectAuthorizationAsync { get; set; } +} + public sealed class NodeClientCreatedEventArgs : EventArgs { public NodeClientCreatedEventArgs(WindowsNodeClient client, string? bearerToken) diff --git a/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs b/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs index dce34289a..fd0058153 100644 --- a/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs +++ b/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs @@ -17,6 +17,25 @@ public static bool TryResolve( string? effectiveGatewayUrl, string? legacyToken, string? legacyBootstrapToken, + out InteractiveGatewayCredential? credential) => + TryResolve( + registry, + settingsDirectory, + identityReader, + effectiveGatewayUrl, + legacyToken, + legacyBootstrapToken, + authorizeCredential: null, + out credential); + + public static bool TryResolve( + GatewayRegistry? registry, + string settingsDirectory, + IDeviceIdentityReader identityReader, + string? effectiveGatewayUrl, + string? legacyToken, + string? legacyBootstrapToken, + Func? authorizeCredential, out InteractiveGatewayCredential? credential) { ArgumentException.ThrowIfNullOrWhiteSpace(settingsDirectory); @@ -30,6 +49,16 @@ public static bool TryResolve( // is for HTTP ?token= auth which the chat/dashboard endpoints expect. if (!string.IsNullOrWhiteSpace(active.SharedGatewayToken)) { + var sharedCredential = new GatewayCredential( + active.SharedGatewayToken!, + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken); + if (authorizeCredential is not null && + !authorizeCredential(active, sharedCredential)) + { + credential = null; + return false; + } credential = new InteractiveGatewayCredential( active.Url, active.SharedGatewayToken!, @@ -43,6 +72,12 @@ public static bool TryResolve( var resolved = resolver.ResolveOperator(active, registry!.GetIdentityDirectory(active.Id)); if (resolved != null) { + if (authorizeCredential is not null && + !authorizeCredential(active, resolved)) + { + credential = null; + return false; + } credential = new InteractiveGatewayCredential( active.Url, resolved.Token, @@ -69,6 +104,7 @@ public static bool TryResolve( { Id = "legacy-settings", Url = gatewayUrl, + IsLocal = GatewayRecordEditing.IsLoopbackEndpoint(gatewayUrl), SharedGatewayToken = legacyToken, BootstrapToken = legacyBootstrapToken }; @@ -79,6 +115,12 @@ public static bool TryResolve( credential = null; return false; } + if (authorizeCredential is not null && + !authorizeCredential(legacyRecord, legacyCredential)) + { + credential = null; + return false; + } credential = new InteractiveGatewayCredential( gatewayUrl, diff --git a/src/OpenClaw.Connection/ManagedLocalGatewayPortProvenanceService.cs b/src/OpenClaw.Connection/ManagedLocalGatewayPortProvenanceService.cs new file mode 100644 index 000000000..300c381f0 --- /dev/null +++ b/src/OpenClaw.Connection/ManagedLocalGatewayPortProvenanceService.cs @@ -0,0 +1,705 @@ +using System; +using System.Collections.Generic; +using System.Collections.Concurrent; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Net; +using System.Threading; +using System.Threading.Tasks; +using System.Xml.Linq; +using OpenClaw.Shared; + +namespace OpenClaw.Connection; + +public enum ManagedLocalPortConflictRepairOutcome +{ + NotNeeded, + Repaired, + BlockedUnknownOwner, + Failed, +} + +public sealed record ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome Outcome, + string? Detail = null); + +internal interface IManagedLocalGatewayPortPlatform +{ + WindowsTcpListenerSnapshotResult CaptureListeners(); + string? GetProcessCommandLine(int processId); + bool IsTrustedWslRelayBinary(string processPath); + bool IsExpectedWslGatewayListening(string distroName, int port); + string? ReadScheduledTaskXml(string taskName); + string? ReadFile(string path); + Task DisableScheduledTaskAsync(string taskName, CancellationToken cancellationToken); + Task EndScheduledTaskAsync(string taskName, CancellationToken cancellationToken); + Task StopProcessAsync( + int processId, + DateTime? expectedStartTimeUtc, + CancellationToken cancellationToken); +} + +internal sealed class WindowsManagedLocalGatewayPortPlatform : IManagedLocalGatewayPortPlatform +{ + public WindowsTcpListenerSnapshotResult CaptureListeners() => + WindowsTcpListenerSnapshot.Capture(); + + public string? GetProcessCommandLine(int processId) => + WindowsTcpListenerSnapshot.GetProcessCommandLine(processId); + + public bool IsTrustedWslRelayBinary(string processPath) + { + try + { + var fullPath = Path.GetFullPath(processPath); + var programFiles = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles); + var systemRoot = Environment.GetFolderPath(Environment.SpecialFolder.Windows); + var canonicalProgramFiles = Path.Combine(programFiles, "WSL", "wslrelay.exe"); + var canonicalSystem = Path.Combine(systemRoot, "System32", "wslrelay.exe"); + var windowsAppsRoot = Path.Combine(programFiles, "WindowsApps") + Path.DirectorySeparatorChar; + var isCanonical = + string.Equals(fullPath, canonicalProgramFiles, StringComparison.OrdinalIgnoreCase) || + string.Equals(fullPath, canonicalSystem, StringComparison.OrdinalIgnoreCase) || + (fullPath.StartsWith(windowsAppsRoot, StringComparison.OrdinalIgnoreCase) && + string.Equals(Path.GetFileName(fullPath), "wslrelay.exe", StringComparison.OrdinalIgnoreCase)); + if (!isCanonical) + return false; + + for (var attempt = 0; attempt < 2; attempt++) + { + var psi = CreateWslRelaySignatureProbe(fullPath); + using var process = Process.Start(psi); + if (process is null) + return false; + if (process.WaitForExit(5_000)) + return process.ExitCode == 0; + + try { process.Kill(entireProcessTree: true); } catch { } + } + return false; + } + catch + { + return false; + } + } + + internal static ProcessStartInfo CreateWslRelaySignatureProbe(string fullPath) + { + var startInfo = new ProcessStartInfo + { + FileName = "powershell.exe", + UseShellExecute = false, + CreateNoWindow = true, + }; + + // A pwsh parent can prepend PowerShell 7 modules that Windows PowerShell + // 5.1 cannot load. Let the child rebuild its native module path so the + // built-in Authenticode cmdlet remains available. + startInfo.Environment.Remove("PSModulePath"); + startInfo.Environment["OPENCLAW_VERIFY_PATH"] = fullPath; + startInfo.ArgumentList.Add("-NoProfile"); + startInfo.ArgumentList.Add("-NonInteractive"); + startInfo.ArgumentList.Add("-Command"); + startInfo.ArgumentList.Add( + "$s=Get-AuthenticodeSignature -LiteralPath $env:OPENCLAW_VERIFY_PATH; " + + "if($s.Status -eq 'Valid' -and $s.SignerCertificate.Subject -match 'Microsoft'){exit 0}; exit 1"); + return startInfo; + } + + public bool IsExpectedWslGatewayListening(string distroName, int port) + { + try + { + var probe = CreateExpectedWslGatewayProbe(distroName, port); + using var process = Process.Start(probe.StartInfo); + if (process is null) + return false; + process.StandardInput.Write(probe.StandardInput); + process.StandardInput.Close(); + if (!process.WaitForExit(5_000)) + { + try { process.Kill(entireProcessTree: true); } catch { } + return false; + } + return process.ExitCode == 0; + } + catch + { + return false; + } + } + + internal static (ProcessStartInfo StartInfo, string StandardInput) CreateExpectedWslGatewayProbe( + string distroName, + int port) + { + var startInfo = new ProcessStartInfo + { + FileName = ResolveWslExePath(), + RedirectStandardInput = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + startInfo.ArgumentList.Add("-d"); + startInfo.ArgumentList.Add(distroName); + startInfo.ArgumentList.Add("--"); + startInfo.ArgumentList.Add("bash"); + startInfo.ArgumentList.Add("-s"); + + // WSL relay can project one distro socket onto both Windows loopback + // families. Verify the service-owned port inside the expected distro + // without assuming family parity across that relay boundary. + var script = + "pid=$(systemctl --user show openclaw-gateway -p MainPID --value 2>/dev/null); " + + "test \"${pid:-0}\" -gt 0 && " + + $"ss -ltnp 2>/dev/null | grep -E ':{port}([^0-9]|$)' | " + + "grep -F \"pid=$pid,\" >/dev/null"; + return (startInfo, script); + } + + public string? ReadScheduledTaskXml(string taskName) => + RunSchtasks(["/Query", "/TN", taskName, "/XML"], CancellationToken.None).GetAwaiter().GetResult().Output; + + public string? ReadFile(string path) + { + try { return File.Exists(path) ? File.ReadAllText(path) : null; } + catch { return null; } + } + + public async Task DisableScheduledTaskAsync(string taskName, CancellationToken cancellationToken) + { + var result = await RunSchtasks(["/Change", "/TN", taskName, "/Disable"], cancellationToken).ConfigureAwait(false); + return result.Success; + } + + public async Task EndScheduledTaskAsync(string taskName, CancellationToken cancellationToken) + { + _ = await RunSchtasks(["/End", "/TN", taskName], cancellationToken).ConfigureAwait(false); + } + + public async Task StopProcessAsync( + int processId, + DateTime? expectedStartTimeUtc, + CancellationToken cancellationToken) + { + try + { + using var process = Process.GetProcessById(processId); + if (expectedStartTimeUtc is { } expected && + process.StartTime.ToUniversalTime() != expected) + { + return false; + } + process.Kill(entireProcessTree: true); + await process.WaitForExitAsync(cancellationToken).ConfigureAwait(false); + return true; + } + catch (ArgumentException) + { + return true; // already exited + } + catch + { + return false; + } + } + + private static async Task<(bool Success, string? Output)> RunSchtasks( + IReadOnlyList arguments, + CancellationToken cancellationToken) + { + Process? process = null; + try + { + var psi = new ProcessStartInfo + { + FileName = WindowsStartupTaskRegistration.ResolveSchtasksPath(), + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + foreach (var argument in arguments) + psi.ArgumentList.Add(argument); + + process = Process.Start(psi); + if (process is null) + return (false, null); + + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(10)); + var stdout = process.StandardOutput.ReadToEndAsync(timeout.Token); + var stderr = process.StandardError.ReadToEndAsync(timeout.Token); + await process.WaitForExitAsync(timeout.Token).ConfigureAwait(false); + var output = (await stdout.ConfigureAwait(false)) + (await stderr.ConfigureAwait(false)); + return (process.ExitCode == 0, output); + } + + catch (OperationCanceledException) + { + try { process?.Kill(entireProcessTree: true); } catch { } + if (cancellationToken.IsCancellationRequested) + throw; + return (false, null); + } + catch + { + return (false, null); + } + finally + { + process?.Dispose(); + } + } + + private static string ResolveWslExePath() + { + var windowsDir = Environment.GetFolderPath(Environment.SpecialFolder.Windows); + if (string.IsNullOrWhiteSpace(windowsDir)) + windowsDir = Environment.GetEnvironmentVariable("SystemRoot") ?? @"C:\Windows"; + return Path.Combine(windowsDir, "System32", "wsl.exe"); + } +} + +/// +/// Proves which process owns a managed WSL gateway's loopback endpoint. Unknown owners are never +/// terminated and never trusted with shared/bootstrap credential fallback. +/// +public sealed class ManagedLocalGatewayPortProvenanceService +{ + private readonly IManagedLocalGatewayPortPlatform _platform; + private readonly IOpenClawLogger _logger; + private readonly Func _getUserProfilePath; + private readonly ConcurrentDictionary + _lastProvenance = new(); + private readonly record struct ProvenanceCacheKey(string GatewayId, string Url); + + internal ManagedLocalGatewayPortProvenanceService( + IManagedLocalGatewayPortPlatform platform, + IOpenClawLogger logger, + Func? getUserProfilePath = null) + { + _platform = platform; + _logger = logger; + _getUserProfilePath = getUserProfilePath ?? + (() => Environment.GetFolderPath(Environment.SpecialFolder.UserProfile)); + } + + public ManagedLocalGatewayPortProvenanceService(IOpenClawLogger logger) + : this(new WindowsManagedLocalGatewayPortPlatform(), logger) + { + } + + public Task InspectAsync( + GatewayRecord record, + CancellationToken cancellationToken = default) => + Task.Run(() => Inspect(record), cancellationToken); + + public GatewayEndpointProvenance Inspect(GatewayRecord record) + { + var result = InspectCore(record); + _lastProvenance[new ProvenanceCacheKey(record.Id, record.Url)] = result; + return result; + } + + public bool IsStrongCredentialAllowed(GatewayRecord record, GatewayCredential credential) + { + var isStrong = + credential.IsBootstrapToken || + string.Equals(credential.Source, CredentialResolver.SourceSharedGatewayToken, StringComparison.Ordinal) || + string.Equals(credential.Source, CredentialResolver.SourceBootstrapToken, StringComparison.Ordinal); + var needsProof = + isStrong && + record.SshTunnel is null && + (record.IsLocal || GatewayRecordEditing.ResolveManagedDistroName(record) is not null) && + GatewayRecordEditing.IsLoopbackEndpoint(record.Url); + if (!needsProof) + return true; + + if (!_lastProvenance.TryGetValue( + new ProvenanceCacheKey(record.Id, record.Url), + out var cached) || + cached.Kind != GatewayEndpointProvenanceKind.ExpectedManagedGateway || + cached.ProcessId is not int expectedPid || + cached.ProcessStartTimeUtc is not DateTime expectedStart || + string.IsNullOrWhiteSpace(cached.ProcessPath) || + !Uri.TryCreate(record.Url, UriKind.Absolute, out var uri) || + GatewayRecordEditing.ResolveManagedDistroName(record) is not { } managedDistroName) + { + return false; + } + + var currentSnapshot = _platform.CaptureListeners(); + if (!HasRequiredAddressFamilies(currentSnapshot, uri)) + return false; + var current = currentSnapshot.Listeners + .Where(listener => listener.Port == uri.Port && AcceptsHost(listener.Address, uri.Host)) + .ToArray(); + if (current.Length == 0 || + !current.All(listener => + listener.ProcessId == expectedPid && + listener.ProcessStartTimeUtc == expectedStart && + string.Equals( + listener.ProcessPath, + cached.ProcessPath, + StringComparison.OrdinalIgnoreCase))) + { + return false; + } + + return _platform.IsExpectedWslGatewayListening(managedDistroName, uri.Port); + } + + private GatewayEndpointProvenance InspectCore(GatewayRecord record) + { + if (record.SshTunnel is not null || + GatewayRecordEditing.ResolveManagedDistroName(record) is not { } managedDistroName || + !Uri.TryCreate(record.Url, UriKind.Absolute, out var uri) || + !uri.IsLoopback || + uri.Port is < 1 or > 65535) + { + return new GatewayEndpointProvenance(GatewayEndpointProvenanceKind.NotApplicable, 0); + } + + var snapshot = _platform.CaptureListeners(); + if (!HasRequiredAddressFamilies(snapshot, uri)) + { + return new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + uri.Port, + Detail: "Windows TCP listener ownership could not be captured completely."); + } + + var listeners = snapshot.Listeners + .Where(listener => listener.Port == uri.Port && AcceptsHost(listener.Address, uri.Host)) + .ToArray(); + if (listeners.Length == 0) + return new GatewayEndpointProvenance(GatewayEndpointProvenanceKind.NoListener, uri.Port); + + var relayTrustByPath = listeners + .Where(listener => + string.Equals(listener.ProcessName, "wslrelay", StringComparison.OrdinalIgnoreCase) && + listener.ProcessPath is not null) + .Select(listener => listener.ProcessPath!) + .Distinct(StringComparer.OrdinalIgnoreCase) + .ToDictionary( + path => path, + path => _platform.IsTrustedWslRelayBinary(path), + StringComparer.OrdinalIgnoreCase); + var expectedDistroListening = + relayTrustByPath.Values.Any(trusted => trusted) && + _platform.IsExpectedWslGatewayListening(managedDistroName, uri.Port); + var classified = listeners + .Select(listener => ClassifyListener( + managedDistroName, + uri.Port, + listener, + relayTrustByPath, + expectedDistroListening)) + .ToArray(); + + if (!ListenerSnapshotStillCurrent(listeners, uri)) + { + return new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + uri.Port, + Detail: "Loopback listener ownership changed during provenance verification."); + } + + if (classified.All(item => item.Kind == GatewayEndpointProvenanceKind.ExpectedManagedGateway)) + { + var first = classified[0]; + return first with { Detail = $"Expected WSL relay owns loopback port {uri.Port}." }; + } + + if (classified.All(item => item.Kind == GatewayEndpointProvenanceKind.ConflictingOpenClawGateway) && + classified.Select(item => item.ProcessId).Distinct().Count() == 1) + { + return classified[0]; + } + + var ownerSummary = string.Join( + "; ", + classified + .Where(item => item.Kind != GatewayEndpointProvenanceKind.ExpectedManagedGateway) + .Select(item => + $"{item.ProcessName ?? "unknown"} (PID {item.ProcessId?.ToString() ?? "?"}): " + + (item.Detail ?? "listener verification failed"))); + return new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + uri.Port, + Detail: $"Loopback port {uri.Port} listener verification failed: {ownerSummary}."); + } + + public async Task RepairConflictAsync( + GatewayRecord record, + CancellationToken cancellationToken, + Func? canContinue = null) + { + canContinue ??= static () => true; + var proof = Inspect(record); + if (proof.Kind == GatewayEndpointProvenanceKind.ExpectedManagedGateway || + proof.Kind == GatewayEndpointProvenanceKind.NoListener || + proof.Kind == GatewayEndpointProvenanceKind.NotApplicable) + { + return new ManagedLocalPortConflictRepairResult(ManagedLocalPortConflictRepairOutcome.NotNeeded); + } + + if (proof.Kind != GatewayEndpointProvenanceKind.ConflictingOpenClawGateway || + proof.ProcessId is not int processId || + string.IsNullOrWhiteSpace(proof.ScheduledTaskName)) + { + return new ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome.BlockedUnknownOwner, + proof.Detail); + } + if (!canContinue()) + return AbortedByIntent(); + + // Re-prove immediately before every destructive action. + var current = Inspect(record); + if (!SameProcessIdentity(current, proof) || + !string.Equals(current.ScheduledTaskName, proof.ScheduledTaskName, StringComparison.Ordinal)) + { + return new ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome.BlockedUnknownOwner, + "Listener ownership changed before repair; no process was stopped."); + } + if (!canContinue()) + return AbortedByIntent(); + + if (!await _platform.DisableScheduledTaskAsync(proof.ScheduledTaskName!, cancellationToken).ConfigureAwait(false)) + { + return new ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome.Failed, + $"Could not disable scheduled task '{proof.ScheduledTaskName}'."); + } + + if (!canContinue()) + return AbortedByIntent(); + await _platform.EndScheduledTaskAsync(proof.ScheduledTaskName!, cancellationToken).ConfigureAwait(false); + + current = Inspect(record); + if (current.Kind is GatewayEndpointProvenanceKind.NoListener or + GatewayEndpointProvenanceKind.ExpectedManagedGateway) + { + return new ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome.Repaired, + "The obsolete native task released the managed WSL gateway port."); + } + + if (!SameProcessIdentity(current, proof)) + { + return new ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome.BlockedUnknownOwner, + "Listener ownership changed after the task was disabled; no process was stopped."); + } + if (!canContinue()) + return AbortedByIntent(); + + if (!await _platform.StopProcessAsync( + processId, + proof.ProcessStartTimeUtc, + cancellationToken).ConfigureAwait(false)) + { + return new ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome.Failed, + $"Could not stop the proven obsolete OpenClaw gateway process (PID {processId})."); + } + + current = Inspect(record); + if (current.Kind is not (GatewayEndpointProvenanceKind.NoListener or + GatewayEndpointProvenanceKind.ExpectedManagedGateway)) + { + return new ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome.Failed, + "The proven process stopped, but the managed gateway port is still owned by an unverified listener."); + } + + _logger.Info($"[GatewayPort] Disabled '{proof.ScheduledTaskName}' and stopped proven obsolete native gateway PID {processId}."); + return new ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome.Repaired, + "Removed a proven obsolete native OpenClaw gateway from the managed WSL gateway port."); + } + + private static ManagedLocalPortConflictRepairResult AbortedByIntent() => + new( + ManagedLocalPortConflictRepairOutcome.BlockedUnknownOwner, + "Gateway changed or was explicitly disconnected before port-conflict repair completed."); + + private GatewayEndpointProvenance ClassifyListener( + string managedDistroName, + int port, + WindowsTcpListenerInfo listener, + IReadOnlyDictionary relayTrustByPath, + bool expectedDistroListening) + { + var isWslRelay = + string.Equals(listener.ProcessName, "wslrelay", StringComparison.OrdinalIgnoreCase); + var relayPath = listener.ProcessPath; + var trustedRelay = + relayPath is not null && + relayTrustByPath.TryGetValue(relayPath, out var trusted) && + trusted; + if (isWslRelay && trustedRelay && expectedDistroListening) + { + return new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ExpectedManagedGateway, + port, + listener.ProcessId, + listener.ProcessName, + listener.ProcessStartTimeUtc, + listener.ProcessPath); + } + + var taskName = $"OpenClaw Gateway ({managedDistroName})"; + if (IsProvenObsoleteNativeGateway(managedDistroName, port, listener, taskName)) + { + return new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ConflictingOpenClawGateway, + port, + listener.ProcessId, + listener.ProcessName, + listener.ProcessStartTimeUtc, + listener.ProcessPath, + taskName, + $"Proven obsolete native OpenClaw gateway '{taskName}' owns loopback port {port}."); + } + + var detail = isWslRelay + ? relayPath is null + ? "WSL relay executable path could not be read." + : !trustedRelay + ? "WSL relay is not the canonical Microsoft-signed binary." + : $"Expected distro '{managedDistroName}' does not report its systemd gateway MainPID owning port {port}." + : "Process is not a verified managed WSL relay or proven obsolete OpenClaw gateway."; + return new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + port, + listener.ProcessId, + listener.ProcessName, + listener.ProcessStartTimeUtc, + listener.ProcessPath, + Detail: detail); + } + + private bool IsProvenObsoleteNativeGateway( + string managedDistroName, + int port, + WindowsTcpListenerInfo listener, + string taskName) + { + if (!string.Equals(Path.GetFileName(listener.ProcessPath), "node.exe", StringComparison.OrdinalIgnoreCase) || + listener.ProcessStartTimeUtc is null) + return false; + + var commandLine = _platform.GetProcessCommandLine(listener.ProcessId); + if (string.IsNullOrWhiteSpace(commandLine) || + !commandLine.Contains(@"\OpenClawTray\native-cli\", StringComparison.OrdinalIgnoreCase) || + !commandLine.Contains(@"\openclaw\dist\index.js", StringComparison.OrdinalIgnoreCase) || + !commandLine.Contains(" gateway ", StringComparison.OrdinalIgnoreCase) || + !commandLine.Contains($"--port {port}", StringComparison.OrdinalIgnoreCase)) + { + return false; + } + + var profileDir = Path.Combine( + _getUserProfilePath(), + $".openclaw-{managedDistroName}"); + var vbsPath = Path.Combine(profileDir, "gateway.vbs"); + var cmdPath = Path.Combine(profileDir, "gateway.cmd"); + var taskXml = _platform.ReadScheduledTaskXml(taskName); + var vbs = _platform.ReadFile(vbsPath); + var cmd = _platform.ReadFile(cmdPath); + if (string.IsNullOrWhiteSpace(taskXml) || + string.IsNullOrWhiteSpace(vbs) || + string.IsNullOrWhiteSpace(cmd)) + { + return false; + } + + string? taskCommand; + try + { + taskCommand = XDocument.Parse(taskXml) + .Descendants() + .FirstOrDefault(element => + string.Equals(element.Name.LocalName, "Command", StringComparison.OrdinalIgnoreCase)) + ?.Value; + } + catch + { + return false; + } + + try + { + return string.Equals( + Path.GetFullPath(taskCommand ?? string.Empty), + Path.GetFullPath(vbsPath), + StringComparison.OrdinalIgnoreCase) && + vbs.Contains(cmdPath, StringComparison.OrdinalIgnoreCase) && + cmd.Contains($"OPENCLAW_STATE_DIR={profileDir}", StringComparison.OrdinalIgnoreCase) && + cmd.Contains($"OPENCLAW_WINDOWS_TASK_NAME={taskName}", StringComparison.OrdinalIgnoreCase) && + cmd.Contains($"OPENCLAW_GATEWAY_PORT={port}", StringComparison.OrdinalIgnoreCase) && + cmd.Contains(@"\OpenClawTray\native-cli\", StringComparison.OrdinalIgnoreCase) && + cmd.Contains(" gateway --port ", StringComparison.OrdinalIgnoreCase); + } + catch + { + return false; + } + } + + private static bool SameProcessIdentity( + GatewayEndpointProvenance left, + GatewayEndpointProvenance right) => + left.Kind == GatewayEndpointProvenanceKind.ConflictingOpenClawGateway && + left.ProcessId == right.ProcessId && + left.ProcessStartTimeUtc == right.ProcessStartTimeUtc; + + private static bool AcceptsHost(IPAddress listenerAddress, string host) + { + if (listenerAddress.Equals(IPAddress.Any) || listenerAddress.Equals(IPAddress.IPv6Any)) + return true; + if (string.Equals(host, "localhost", StringComparison.OrdinalIgnoreCase)) + return IPAddress.IsLoopback(listenerAddress); + return IPAddress.TryParse(host, out var target) && listenerAddress.Equals(target); + } + + private bool ListenerSnapshotStillCurrent( + IReadOnlyList original, + Uri uri) + { + var snapshot = _platform.CaptureListeners(); + if (!HasRequiredAddressFamilies(snapshot, uri)) + return false; + var current = snapshot.Listeners + .Where(listener => listener.Port == uri.Port && AcceptsHost(listener.Address, uri.Host)) + .ToArray(); + if (current.Length != original.Count) + return false; + + return original.All(expected => current.Any(actual => + actual.Address.Equals(expected.Address) && + actual.Port == expected.Port && + actual.ProcessId == expected.ProcessId && + actual.ProcessStartTimeUtc == expected.ProcessStartTimeUtc && + string.Equals(actual.ProcessPath, expected.ProcessPath, StringComparison.OrdinalIgnoreCase))); + } + + private static bool HasRequiredAddressFamilies( + WindowsTcpListenerSnapshotResult snapshot, + Uri uri) + { + if (string.Equals(uri.Host, "localhost", StringComparison.OrdinalIgnoreCase)) + return snapshot.Ipv4Complete && snapshot.Ipv6Complete; + if (!IPAddress.TryParse(uri.Host, out var address)) + return false; + return address.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork + ? snapshot.Ipv4Complete + : snapshot.Ipv6Complete; + } +} diff --git a/src/OpenClaw.Connection/NodeConnector.cs b/src/OpenClaw.Connection/NodeConnector.cs index 510e274ef..2d7561268 100644 --- a/src/OpenClaw.Connection/NodeConnector.cs +++ b/src/OpenClaw.Connection/NodeConnector.cs @@ -7,7 +7,7 @@ namespace OpenClaw.Connection; /// Capability setup (canvas, screen capture, etc.) is handled by NodeService, /// which has WinUI dependencies and remains in App.xaml.cs for now. /// -public sealed class NodeConnector : INodeConnector, INodeConnectorTelemetryEvents +public sealed class NodeConnector : INodeConnector, INodeConnectorTelemetryEvents, INodeConnectorReconnectPolicy { private readonly IOpenClawLogger _logger; private readonly ConnectionDiagnostics? _diagnostics; @@ -16,6 +16,7 @@ public sealed class NodeConnector : INodeConnector, INodeConnectorTelemetryEvent private WindowsNodeClient? _client; private long _clientGeneration; private bool _disposed; + public Func>? ReconnectAuthorizationAsync { get; set; } public event EventHandler? StatusChanged; public event EventHandler? PairingStatusChanged; @@ -101,6 +102,7 @@ private async Task ConnectCoreAsync( identityPath, nodeLogger, bootstrapToken: credential.IsBootstrapToken ? credential.Token : null); + client.ReconnectAuthorizationAsync = ReconnectAuthorizationAsync; // Share v2 signature flag from operator — avoid wasting a roundtrip on v3 if (useV2Signature) diff --git a/src/OpenClaw.Connection/WindowsTcpListenerSnapshot.cs b/src/OpenClaw.Connection/WindowsTcpListenerSnapshot.cs new file mode 100644 index 000000000..004b3ea14 --- /dev/null +++ b/src/OpenClaw.Connection/WindowsTcpListenerSnapshot.cs @@ -0,0 +1,237 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Net; +using System.Runtime.InteropServices; + +namespace OpenClaw.Connection; + +public sealed record WindowsTcpListenerInfo( + IPAddress Address, + int Port, + int ProcessId, + string? ProcessName, + string? ProcessPath, + DateTime? ProcessStartTimeUtc = null); + +public sealed record WindowsTcpListenerSnapshotResult( + IReadOnlyList Listeners, + bool Ipv4Complete, + bool Ipv6Complete); + +/// Address-specific TCP listener ownership from the Windows IP Helper API. +public static class WindowsTcpListenerSnapshot +{ + public static WindowsTcpListenerSnapshotResult Capture() + { + if (!OperatingSystem.IsWindows()) + return new([], Ipv4Complete: false, Ipv6Complete: false); + + var result = new List(); + var ipv4Complete = CaptureIpv4(result); + var ipv6Complete = CaptureIpv6(result); + return new(result, ipv4Complete, ipv6Complete); + } + + public static string? GetProcessCommandLine(int processId) + { + if (processId <= 0) + return null; + + try + { + var psi = new ProcessStartInfo( + "powershell.exe", + $"-NoProfile -Command \"(Get-CimInstance Win32_Process -Filter 'ProcessId={processId}').CommandLine\"") + { + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + using var process = Process.Start(psi); + if (process is null) + return null; + + var readTask = process.StandardOutput.ReadToEndAsync(); + if (!process.WaitForExit(5_000)) + { + try { process.Kill(entireProcessTree: true); } catch { } + return null; + } + + return readTask.GetAwaiter().GetResult().Trim(); + } + catch + { + return null; + } + } + + private static bool CaptureIpv4(List destination) + { + return CaptureTable( + AfInet, + Marshal.SizeOf(), + rowPtr => + { + var row = Marshal.PtrToStructure(rowPtr); + var address = new IPAddress(BitConverter.GetBytes(row.LocalAddress)); + return (address, ReadPort(row.LocalPort), unchecked((int)row.OwningProcessId)); + }, + destination); + } + + private static bool CaptureIpv6(List destination) + { + return CaptureTable( + AfInet6, + Marshal.SizeOf(), + rowPtr => + { + var row = Marshal.PtrToStructure(rowPtr); + var address = new IPAddress(row.LocalAddress, row.LocalScopeId); + return (address, ReadPort(row.LocalPort), unchecked((int)row.OwningProcessId)); + }, + destination); + } + + private static bool CaptureTable( + int addressFamily, + int rowSize, + Func readRow, + List destination) + { + for (var attempt = 0; attempt < 3; attempt++) + { + var bufferLength = 0; + var status = GetExtendedTcpTable( + IntPtr.Zero, + ref bufferLength, + sort: true, + ipVersion: addressFamily, + tableClass: TcpTableOwnerPidListener, + reserved: 0); + if (status != ErrorInsufficientBuffer || bufferLength <= 0) + return false; + + var tablePtr = Marshal.AllocHGlobal(bufferLength); + try + { + status = GetExtendedTcpTable( + tablePtr, + ref bufferLength, + sort: true, + ipVersion: addressFamily, + tableClass: TcpTableOwnerPidListener, + reserved: 0); + if (status == ErrorInsufficientBuffer) + continue; // listener table grew between size/read calls + if (status != ErrorSuccess) + return false; + + var rowCount = Marshal.ReadInt32(tablePtr); + var rowPtr = IntPtr.Add(tablePtr, sizeof(int)); + var captured = new List(rowCount); + for (var i = 0; i < rowCount; i++) + { + var row = readRow(rowPtr); + if (row.Port is >= 1 and <= 65535) + { + ResolveProcess( + row.ProcessId, + out var processName, + out var processPath, + out var processStartTimeUtc); + captured.Add(new WindowsTcpListenerInfo( + row.Address, + row.Port, + row.ProcessId, + processName, + processPath, + processStartTimeUtc)); + } + rowPtr = IntPtr.Add(rowPtr, rowSize); + } + destination.AddRange(captured); + return true; + } + finally + { + Marshal.FreeHGlobal(tablePtr); + } + } + return false; + } + + private static int ReadPort(byte[] bytes) => + bytes is { Length: >= 2 } ? (bytes[0] << 8) + bytes[1] : 0; + + private static void ResolveProcess( + int processId, + out string? processName, + out string? processPath, + out DateTime? processStartTimeUtc) + { + processName = null; + processPath = null; + processStartTimeUtc = null; + if (processId <= 0) + return; + + try + { + using var process = Process.GetProcessById(processId); + processName = process.ProcessName; + try { processPath = process.MainModule?.FileName; } catch { } + try { processStartTimeUtc = process.StartTime.ToUniversalTime(); } catch { } + } + catch + { + } + } + + private const int AfInet = 2; + private const int AfInet6 = 23; + private const int TcpTableOwnerPidListener = 3; + private const uint ErrorSuccess = 0; + private const uint ErrorInsufficientBuffer = 122; + + [DllImport("iphlpapi.dll", SetLastError = true)] + private static extern uint GetExtendedTcpTable( + IntPtr tcpTable, + ref int tcpTableLength, + bool sort, + int ipVersion, + int tableClass, + uint reserved); + + [StructLayout(LayoutKind.Sequential)] + private struct MibTcpRowOwnerPid + { + public uint State; + public uint LocalAddress; + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] + public byte[] LocalPort; + public uint RemoteAddress; + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] + public byte[] RemotePort; + public uint OwningProcessId; + } + + [StructLayout(LayoutKind.Sequential)] + private struct MibTcp6RowOwnerPid + { + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)] + public byte[] LocalAddress; + public uint LocalScopeId; + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] + public byte[] LocalPort; + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 16)] + public byte[] RemoteAddress; + public uint RemoteScopeId; + [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] + public byte[] RemotePort; + public uint State; + public uint OwningProcessId; + } +} diff --git a/src/OpenClaw.Connection/WslGatewayController.cs b/src/OpenClaw.Connection/WslGatewayController.cs index b28dd3824..7747a952e 100644 --- a/src/OpenClaw.Connection/WslGatewayController.cs +++ b/src/OpenClaw.Connection/WslGatewayController.cs @@ -98,4 +98,33 @@ public async Task RunAsync( result.StandardOutput, result.StandardError); } + + /// + /// Forcefully restarts a distro that could not be controlled in-place: it issues a host-side + /// wsl.exe --terminate (which succeeds even when the distro/VM is wedged and cannot run an + /// in-distro command), then cold-restarts the gateway. The follow-up in-distro command re-boots a + /// fresh VM (systemd), so this recovers a hung WSL instance that a plain gateway restart + /// cannot reach. Used only as an escalation after an in-place restart fails. + /// + public async Task ForceRestartAsync( + string distroName, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(distroName)) + { + throw new ArgumentException("WSL distro name is required.", nameof(distroName)); + } + + var normalizedDistroName = distroName.Trim(); + logger.Info($"Force-terminating WSL distro '{normalizedDistroName}' before restart (in-place restart failed)."); + + // Terminate is best-effort: a "not running" distro still returns success, and any failure here + // should not stop the cold restart below from surfacing the real error. + // slopwatch-ignore: SW003 Terminate is best-effort; its failure cannot improve the caller state and the restart below is authoritative. + try { await commandRunner.TerminateDistroAsync(normalizedDistroName, cancellationToken).ConfigureAwait(false); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } + catch (Exception ex) { logger.Warn($"Force-terminate of '{normalizedDistroName}' failed (continuing to restart): {ex.Message}"); } + + return await RunAsync(normalizedDistroName, WslGatewayControlAction.Restart, cancellationToken).ConfigureAwait(false); + } } diff --git a/src/OpenClaw.SetupEngine.UI/Pages/WizardPage.xaml.cs b/src/OpenClaw.SetupEngine.UI/Pages/WizardPage.xaml.cs index 56d6a4e4e..b3229ff09 100644 --- a/src/OpenClaw.SetupEngine.UI/Pages/WizardPage.xaml.cs +++ b/src/OpenClaw.SetupEngine.UI/Pages/WizardPage.xaml.cs @@ -150,7 +150,8 @@ private async Task ConnectClientAsync() var record = registry.GetActive() ?? throw new InvalidOperationException("No active gateway record found."); _hostAccessPlan = GatewayHostAccessClassifier.Classify(record); var identityPath = registry.GetIdentityDirectory(record.Id); - var token = DeviceIdentity.TryReadStoredDeviceToken(identityPath) + var deviceToken = DeviceIdentity.TryReadStoredDeviceToken(identityPath); + var token = deviceToken ?? record.SharedGatewayToken ?? record.BootstrapToken ?? throw new InvalidOperationException("No gateway credential found."); @@ -158,10 +159,39 @@ private async Task ConnectClientAsync() // The active record owns the endpoint as well as the credential identity. Resolve // tunnel-backed records to their Windows-side local forward instead of bypassing SSH. var gatewayUrl = GatewayClientEndpointResolver.Resolve(record); + var provenanceService = new ManagedLocalGatewayPortProvenanceService(NullLogger.Instance); + if (deviceToken is null && + record.SshTunnel is null && + GatewayRecordEditing.ResolveManagedDistroName(record) is not null && + GatewayRecordEditing.IsLoopbackEndpoint(record.Url)) + { + var provenance = await provenanceService.InspectAsync(record); + if (provenance.Kind != GatewayEndpointProvenanceKind.ExpectedManagedGateway) + { + throw new InvalidOperationException( + "The managed gateway address is not owned by the verified WSL gateway; no credential was sent."); + } + } var client = new OpenClawGatewayClient(gatewayUrl, token, logger: NullLogger.Instance, identityPath: identityPath) { UseV2Signature = true }; + client.ReconnectAuthorizationAsync = async cancellationToken => + { + if (record.SshTunnel is not null || + GatewayRecordEditing.ResolveManagedDistroName(record) is null || + !GatewayRecordEditing.IsLoopbackEndpoint(record.Url)) + { + return ReconnectAuthorizationResult.AllowedResult; + } + var provenance = await provenanceService.InspectAsync(record, cancellationToken); + return provenance.Kind == GatewayEndpointProvenanceKind.ExpectedManagedGateway + ? ReconnectAuthorizationResult.AllowedResult + : new ReconnectAuthorizationResult( + false, + GatewayErrorKind.LocalPortConflict, + provenance.Detail); + }; var outcome = await WaitForConnectAsync(client, TimeSpan.FromSeconds(20)); if (!outcome) diff --git a/src/OpenClaw.SetupEngine/SetupContext.cs b/src/OpenClaw.SetupEngine/SetupContext.cs index d73234c5c..5057d1a21 100644 --- a/src/OpenClaw.SetupEngine/SetupContext.cs +++ b/src/OpenClaw.SetupEngine/SetupContext.cs @@ -1,6 +1,7 @@ using System.Diagnostics.CodeAnalysis; using System.Text.Json; using System.Text.Json.Serialization; +using OpenClaw.Connection; namespace OpenClaw.SetupEngine; @@ -444,6 +445,8 @@ public sealed class SetupContext public string? WindowsTailnetDnsSuffix { get; set; } public string? TailscaleDnsName { get; set; } public IExternalAuthorizationPresenter? ExternalAuthorizationPresenter { get; set; } + public Func>? + EndpointProvenanceProbe { get; set; } // Data directory for gateway registry and identity files public string DataDir { get; } diff --git a/src/OpenClaw.SetupEngine/SetupSteps.cs b/src/OpenClaw.SetupEngine/SetupSteps.cs index 66b2a851a..f1fd08d7c 100644 --- a/src/OpenClaw.SetupEngine/SetupSteps.cs +++ b/src/OpenClaw.SetupEngine/SetupSteps.cs @@ -1873,6 +1873,9 @@ record = registry.AddOrUpdate(record); var reachability = await WindowsGatewayReachability.VerifyAsync(ctx, "operator", ct); if (!reachability.IsSuccess) return reachability; + var provenanceCheck = await EnsurePairingEndpointTrustedAsync(ctx, ct); + if (provenanceCheck is not null) + return provenanceCheck; // Connect operator WebSocket — handle pairing-required flow var wsLogger = new SetupOpenClawLogger(ctx.Logger); @@ -1882,6 +1885,7 @@ record = registry.AddOrUpdate(record); { // Phase 1: Initial connect (may get PAIRING_REQUIRED) client = new OpenClawGatewayClient(gatewayUrl, token, logger: wsLogger, identityPath: identityPath); + ApplyReconnectAuthorization(client, ctx); client.UseV2Signature = true; // Local gateway uses v2 signature format var phase1Result = await WaitForConnectionOrPairing(client, ctx, TimeSpan.FromSeconds(15), ct); @@ -1911,7 +1915,11 @@ record = registry.AddOrUpdate(record); await Task.Delay(2000, ct); // Phase 2: Reconnect — the device should now be approved + provenanceCheck = await EnsurePairingEndpointTrustedAsync(ctx, ct); + if (provenanceCheck is not null) + return provenanceCheck; client = new OpenClawGatewayClient(gatewayUrl, token, logger: wsLogger, identityPath: identityPath); + ApplyReconnectAuthorization(client, ctx); client.UseV2Signature = true; var phase2Result = await WaitForConnectionOrPairing(client, ctx, TimeSpan.FromSeconds(20), ct); @@ -1949,6 +1957,49 @@ record = registry.AddOrUpdate(record); } } + internal static async Task EnsurePairingEndpointTrustedAsync( + SetupContext ctx, + CancellationToken cancellationToken) + { + var record = new GatewayRecord + { + Id = ctx.GatewayRecordId ?? "setup-managed-gateway", + Url = ctx.GatewayUrl ?? ctx.Config.EffectiveGatewayUrl, + IsLocal = true, + SetupManagedDistroName = ctx.DistroName, + }; + var probe = ctx.EndpointProvenanceProbe ?? + new ManagedLocalGatewayPortProvenanceService( + new SetupOpenClawLogger(ctx.Logger)).InspectAsync; + var provenance = await probe(record, cancellationToken).ConfigureAwait(false); + return provenance.Kind switch + { + GatewayEndpointProvenanceKind.ExpectedManagedGateway or + GatewayEndpointProvenanceKind.NotApplicable => null, + GatewayEndpointProvenanceKind.NoListener => + StepResult.Fail("The managed WSL gateway is not listening; no pairing credential was sent."), + _ => StepResult.Terminal( + provenance.Detail ?? + "The managed gateway address is owned by an unverified process; no pairing credential was sent."), + }; + } + + internal static void ApplyReconnectAuthorization( + WebSocketClientBase client, + SetupContext ctx) + { + client.ReconnectAuthorizationAsync = async cancellationToken => + { + var failure = await EnsurePairingEndpointTrustedAsync(ctx, cancellationToken).ConfigureAwait(false); + return failure is null + ? ReconnectAuthorizationResult.AllowedResult + : new ReconnectAuthorizationResult( + false, + GatewayErrorKind.LocalPortConflict, + failure.Message); + }; + } + /// /// After initial pairing, the gateway knows us via auth.token (shared gateway token). /// The tray will connect using auth.deviceToken (the token we just received). @@ -1978,6 +2029,7 @@ private static async Task FinalizeWithDeviceToken( // Connect exactly as the tray would: pass deviceToken as the credential var finalClient = new OpenClawGatewayClient(gatewayUrl, deviceToken, logger: wsLogger, identityPath: identityPath); + ApplyReconnectAuthorization(finalClient, ctx); finalClient.UseV2Signature = true; try @@ -2007,6 +2059,7 @@ private static async Task FinalizeWithDeviceToken( // One more connect to confirm finalClient = new OpenClawGatewayClient(gatewayUrl, deviceToken, logger: wsLogger, identityPath: identityPath); + ApplyReconnectAuthorization(finalClient, ctx); finalClient.UseV2Signature = true; var finalResult = await WaitForConnectionOrPairing(finalClient, ctx, TimeSpan.FromSeconds(15), ct); @@ -2270,10 +2323,16 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati var reachability = await WindowsGatewayReachability.VerifyAsync(ctx, "node", ct); if (!reachability.IsSuccess) return reachability; + var provenanceCheck = await PairOperatorStep.EnsurePairingEndpointTrustedAsync(ctx, ct); + if (provenanceCheck is not null) + return provenanceCheck; var drainResult = await VerifyEndToEndStep.DrainPendingDeviceApprovalsAsync(ctx, ct); if (!drainResult.IsSuccess) return drainResult; + provenanceCheck = await PairOperatorStep.EnsurePairingEndpointTrustedAsync(ctx, ct); + if (provenanceCheck is not null) + return provenanceCheck; var wsLogger = new SetupOpenClawLogger(ctx.Logger); WindowsNodeClient? client = null; @@ -2282,6 +2341,7 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati { // Phase 1: Connect (may get PAIRING_REQUIRED) client = new WindowsNodeClient(gatewayUrl, token, identityPath, logger: wsLogger); + PairOperatorStep.ApplyReconnectAuthorization(client, ctx); client.UseV2Signature = true; // Register capabilities BEFORE connect — gateway stores them from hello message @@ -2313,7 +2373,11 @@ public override async Task ExecuteAsync(SetupContext ctx, Cancellati await Task.Delay(2000, ct); // Phase 2: Reconnect after approval + provenanceCheck = await PairOperatorStep.EnsurePairingEndpointTrustedAsync(ctx, ct); + if (provenanceCheck is not null) + return provenanceCheck; client = new WindowsNodeClient(gatewayUrl, token, identityPath, logger: wsLogger); + PairOperatorStep.ApplyReconnectAuthorization(client, ctx); client.UseV2Signature = true; RegisterCapabilitiesFromConfig(client, ctx); @@ -2383,6 +2447,7 @@ private static async Task FinalizeNodeWithDeviceToken( await Task.Delay(TimeSpan.FromSeconds(5), ct); var finalClient = new WindowsNodeClient(gatewayUrl, nodeToken, identityPath, logger: wsLogger); + PairOperatorStep.ApplyReconnectAuthorization(finalClient, ctx); finalClient.UseV2Signature = true; try @@ -2409,6 +2474,7 @@ private static async Task FinalizeNodeWithDeviceToken( await Task.Delay(2000, ct); finalClient = new WindowsNodeClient(gatewayUrl, nodeToken, identityPath, logger: wsLogger); + PairOperatorStep.ApplyReconnectAuthorization(finalClient, ctx); finalClient.UseV2Signature = true; var finalResult = await WaitForNodeConnection(finalClient, ctx, TimeSpan.FromSeconds(15), ct); @@ -3430,6 +3496,7 @@ private static async Task FinalizeOperatorForTray( await Task.Delay(TimeSpan.FromSeconds(5), ct); var client = new OpenClawGatewayClient(gatewayUrl, deviceToken, logger: wsLogger, identityPath: identityPath); + PairOperatorStep.ApplyReconnectAuthorization(client, ctx); client.UseV2Signature = true; try @@ -3463,7 +3530,11 @@ private static async Task FinalizeOperatorForTray( // Reconnect with the SHARED GATEWAY TOKEN to get a fresh device token. ctx.Logger.Info("Reconnecting with shared token to get fresh device token after approval"); + var provenanceCheck = await PairOperatorStep.EnsurePairingEndpointTrustedAsync(ctx, ct); + if (provenanceCheck is not null) + return provenanceCheck; client = new OpenClawGatewayClient(gatewayUrl, ctx.SharedGatewayToken!, logger: wsLogger, identityPath: identityPath); + PairOperatorStep.ApplyReconnectAuthorization(client, ctx); client.UseV2Signature = true; var confirmResult = await PairOperatorStep.WaitForConnectionOrPairing(client, ctx, TimeSpan.FromSeconds(15), ct); diff --git a/src/OpenClaw.SetupEngine/SetupWizardRunner.cs b/src/OpenClaw.SetupEngine/SetupWizardRunner.cs index 39a24719e..a921b0992 100644 --- a/src/OpenClaw.SetupEngine/SetupWizardRunner.cs +++ b/src/OpenClaw.SetupEngine/SetupWizardRunner.cs @@ -62,6 +62,9 @@ public async Task RunAsync(CancellationToken ct) try { + var provenanceCheck = await PairOperatorStep.EnsurePairingEndpointTrustedAsync(_ctx, ct); + if (provenanceCheck is not null) + return provenanceCheck; client = CreateWizardClient(credential, identityPath, wsLogger); var connection = await PairOperatorStep.WaitForConnectionOrPairing(client, _ctx, TimeSpan.FromSeconds(20), ct); if (connection == PairOperatorStep.ConnectionOutcome.PairingRequired && _ctx.Config.AutoApprovePairing) @@ -75,6 +78,9 @@ public async Task RunAsync(CancellationToken ct) return approval; await Task.Delay(2000, ct); + provenanceCheck = await PairOperatorStep.EnsurePairingEndpointTrustedAsync(_ctx, ct); + if (provenanceCheck is not null) + return provenanceCheck; client = CreateWizardClient(credential, identityPath, wsLogger); connection = await PairOperatorStep.WaitForConnectionOrPairing(client, _ctx, TimeSpan.FromSeconds(20), ct); } @@ -110,6 +116,10 @@ async Task SendWizardNextAsync(object parameters, int timeoutMs) client!.Dispose(); await Task.Delay(TimeSpan.FromSeconds(3), ct); + var restartProvenance = + await PairOperatorStep.EnsurePairingEndpointTrustedAsync(_ctx, ct); + if (restartProvenance is not null) + throw new WizardFatalException(restartProvenance.Message ?? "Gateway ownership changed."); client = CreateWizardClient(credential, identityPath, wsLogger); var reconnect = await PairOperatorStep.WaitForConnectionOrPairing(client, _ctx, TimeSpan.FromSeconds(30), ct); if (reconnect != PairOperatorStep.ConnectionOutcome.Connected) @@ -266,10 +276,12 @@ async Task SendWizardNextAsync(object parameters, int timeoutMs) private OpenClawGatewayClient CreateWizardClient(string credential, string identityPath, IOpenClawLogger wsLogger) { - return new OpenClawGatewayClient(_ctx.GatewayUrl!, credential, logger: wsLogger, identityPath: identityPath) + var client = new OpenClawGatewayClient(_ctx.GatewayUrl!, credential, logger: wsLogger, identityPath: identityPath) { UseV2Signature = true }; + PairOperatorStep.ApplyReconnectAuthorization(client, _ctx); + return client; } private async Task TryCancelWizardAsync(OpenClawGatewayClient client, string sessionId) diff --git a/src/OpenClaw.Shared/Capabilities/BrowserProxyCapability.cs b/src/OpenClaw.Shared/Capabilities/BrowserProxyCapability.cs index ccac22db3..4cdb2b99b 100644 --- a/src/OpenClaw.Shared/Capabilities/BrowserProxyCapability.cs +++ b/src/OpenClaw.Shared/Capabilities/BrowserProxyCapability.cs @@ -24,6 +24,8 @@ public class BrowserProxyCapability : NodeCapabilityBase private readonly int? _sshTunnelLocalPort; private readonly bool _allowGatewayPortFallback; private readonly HttpClient _httpClient; + private readonly Func>? + _authorizeEndpointAsync; public BrowserProxyCapability( IOpenClawLogger logger, @@ -34,7 +36,9 @@ public BrowserProxyCapability( int? controlPortOverride = null, bool useSshTunnel = false, int? sshTunnelLocalPort = null, - bool? allowGatewayPortFallback = null) : base(logger) + bool? allowGatewayPortFallback = null, + Func>? + authorizeEndpointAsync = null) : base(logger) { _gatewayUrl = gatewayUrl; _bearerToken = bearerToken ?? ""; @@ -44,6 +48,7 @@ public BrowserProxyCapability( _sshTunnelLocalPort = sshTunnelLocalPort; _allowGatewayPortFallback = allowGatewayPortFallback ?? BrowserControlEndpoint.AllowsGatewayPortFallback(gatewayUrl); + _authorizeEndpointAsync = authorizeEndpointAsync; _httpClient = handler == null ? new HttpClient() : new HttpClient(handler); } @@ -72,6 +77,13 @@ public override async Task ExecuteAsync(NodeInvokeRequest re var uri = BuildUri(controlPort, path, request.Args); try { + if (!string.IsNullOrWhiteSpace(_bearerToken) && + _authorizeEndpointAsync is not null && + !await _authorizeEndpointAsync(uri, timeoutCts.Token).ConfigureAwait(false)) + { + return Error("Browser control authentication was blocked because the local listener owner could not be verified."); + } + using var httpRequest = CreateHttpRequest(method, uri, request.Args, usePasswordAuth: false); using var response = await _httpClient.SendAsync(httpRequest, timeoutCts.Token); var responseText = await response.Content.ReadAsStringAsync(timeoutCts.Token); diff --git a/src/OpenClaw.Shared/GatewayErrorClassifier.cs b/src/OpenClaw.Shared/GatewayErrorClassifier.cs index 73eab4137..6d3172f86 100644 --- a/src/OpenClaw.Shared/GatewayErrorClassifier.cs +++ b/src/OpenClaw.Shared/GatewayErrorClassifier.cs @@ -25,6 +25,17 @@ public enum GatewayErrorKind /// TokenDrift, + /// + /// The gateway explicitly rejected this device's stored device token + /// (structured code AUTH_DEVICE_TOKEN_MISMATCH) — distinct from a wrong + /// shared/gateway token. This is the one token failure that is safe to + /// auto-recover: clear only the device token and reconnect, letting a still-valid + /// shared/bootstrap credential re-derive a fresh device token. Broad + /// stays a manual re-pair signal; this exact kind gates + /// automatic recovery. + /// + DeviceTokenMismatch, + /// /// Authenticated but missing a required operator/node scope (e.g. cannot /// approve pairing or read config) — the fix is to re-pair for higher scopes. @@ -43,6 +54,12 @@ public enum GatewayErrorKind /// SSH tunnel could not be established or dropped. Tunnel, + /// + /// A different or unverified local process owns the managed gateway's expected loopback port. + /// Credentials must not be downgraded or disclosed to that listener. + /// + LocalPortConflict, + /// Gateway returned a 5xx / internal error. Server, @@ -88,6 +105,13 @@ public static GatewayErrorKind Classify(string? error) (Contains(e, "forbidden") && Contains(e, "scope"))) return GatewayErrorKind.ScopeMismatch; + // A gateway/shared token mismatch is NOT device-token drift. The gateway emits this + // wording when gateway.remote.token and gateway.auth.token disagree (including when the + // wrong local gateway owns the expected port). Re-pairing the device cannot fix it. + if (Contains(e, "gateway token mismatch") || + (Contains(e, "gateway.remote.token") && Contains(e, "gateway.auth.token"))) + return GatewayErrorKind.Auth; + // Token drift — the device token specifically is stale/unknown. if (Contains(e, "re-pair") || Contains(e, "repair token") || Contains(e, "token rotat") || Contains(e, "token revoked") || @@ -131,4 +155,79 @@ public static GatewayErrorKind Classify(string? error) private static bool Contains(string haystack, string needle) => haystack.Contains(needle, StringComparison.Ordinal); + + public static bool IsSharedGatewayTokenMismatch(string? message) => + !string.IsNullOrWhiteSpace(message) && + (message.Contains(SharedTokenMismatchCode, StringComparison.OrdinalIgnoreCase) || + message.Contains("gateway token mismatch", StringComparison.OrdinalIgnoreCase) || + (message.Contains("gateway.remote.token", StringComparison.OrdinalIgnoreCase) && + message.Contains("gateway.auth.token", StringComparison.OrdinalIgnoreCase))); + + /// The structured gateway code for a stale device token. + public const string DeviceTokenMismatchCode = "AUTH_DEVICE_TOKEN_MISMATCH"; + + /// The structured gateway code for a wrong shared/gateway token. + public const string SharedTokenMismatchCode = "AUTH_TOKEN_MISMATCH"; + + /// + /// Code-aware classification. Structured error codes (top-level error.code and + /// nested error.details.code) are authoritative and are checked BEFORE the textual + /// heuristic, because a gateway may send a generic message (e.g. "unauthorized") with the + /// real reason only in a code. Critically this separates a stale device token + /// (, auto-recoverable) from a wrong + /// shared token (, NOT device-recoverable) — + /// the plain heuristic conflates both into + /// . Falls back to the textual heuristic when no + /// code is authoritative. + /// + public static GatewayErrorKind ClassifyWithCode(string? message, params string?[] codes) + { + if (codes != null) + { + foreach (var code in codes) + { + if (string.IsNullOrWhiteSpace(code)) + continue; + if (string.Equals(code, DeviceTokenMismatchCode, StringComparison.OrdinalIgnoreCase)) + return GatewayErrorKind.DeviceTokenMismatch; + // A wrong shared/gateway token is terminal auth but must NOT be treated as a + // recoverable device-token drift (clearing the device token would just loop). + if (string.Equals(code, SharedTokenMismatchCode, StringComparison.OrdinalIgnoreCase)) + return GatewayErrorKind.Auth; + } + } + + // Message-level exact device phrasing (the structured code carried as text, or an + // explicit "device token mismatch"). A bare "token mismatch" is deliberately NOT matched + // here — it is shared-vs-device ambiguous and falls through to broad TokenDrift below. + if (!string.IsNullOrWhiteSpace(message) && + (message.Contains(DeviceTokenMismatchCode, StringComparison.OrdinalIgnoreCase) || + message.Contains("device token mismatch", StringComparison.OrdinalIgnoreCase))) + return GatewayErrorKind.DeviceTokenMismatch; + + if (!string.IsNullOrWhiteSpace(message) && + message.Contains(SharedTokenMismatchCode, StringComparison.OrdinalIgnoreCase)) + return GatewayErrorKind.Auth; + + // Fall back to the textual heuristic. Fold any remaining (non-token) structured codes into + // the classified text so a code-only signal (e.g. AUTH_RATE_LIMITED carried in the code with + // a generic "unauthorized" message) keeps the precision main had via + // Classify(code + " " + message). The two token codes were already handled authoritatively + // above, so this only affects other codes (rate-limit, server, tls, …). + if (codes != null) + { + var joined = string.Empty; + foreach (var code in codes) + { + if (string.IsNullOrWhiteSpace(code)) + continue; + joined = joined.Length == 0 ? code : $"{joined} {code}"; + } + + if (joined.Length > 0) + return Classify(string.IsNullOrWhiteSpace(message) ? joined : $"{joined} {message}"); + } + + return Classify(message); + } } diff --git a/src/OpenClaw.Shared/OpenClawGatewayClient.cs b/src/OpenClaw.Shared/OpenClawGatewayClient.cs index 5a8273836..7d84f0b88 100644 --- a/src/OpenClaw.Shared/OpenClawGatewayClient.cs +++ b/src/OpenClaw.Shared/OpenClawGatewayClient.cs @@ -244,11 +244,30 @@ protected override void OnDisposing() public event EventHandler? PairingRequired; /// Raised when v3 signature was rejected and client fell back to v2. public event EventHandler? V2SignatureFallback; + /// + /// Raised with the typed reason for an operator connection failure. Consumers should use this + /// kind for policy/UI decisions and keep the accompanying text only for sanitized detail. + /// + public event EventHandler? ConnectionFailure; public string? OperatorDeviceId => _operatorDeviceId; public IReadOnlyList GrantedOperatorScopes => _grantedOperatorScopes; public virtual bool IsConnectedToGateway => IsConnected; + protected override void OnConnectionException(Exception exception) + { + RaiseConnectionFailure(GatewayErrorClassifier.Classify(exception.ToString())); + } + + protected override void OnReconnectAuthorizationDenied( + ReconnectAuthorizationResult authorization) + { + RaiseConnectionFailure(authorization.FailureKind); + } + + protected void RaiseConnectionFailure(GatewayErrorKind kind) => + ConnectionFailure?.Invoke(this, kind); + public OpenClawGatewayClient(string gatewayUrl, string token, IOpenClawLogger? logger = null, bool tokenIsBootstrapToken = false, bool bootstrapPairAsNode = false, string? identityPath = null, bool ignoreStoredDeviceToken = false) : base(gatewayUrl, token, logger) { @@ -2164,6 +2183,7 @@ private void HandleRequestError(string? method, JsonElement root) // v2 also rejected — real auth error _logger.Warn($"[HANDSHAKE] v2 signature also rejected — wrong key or token. Raw: {message}"); _authFailed = true; + RaiseConnectionFailure(GatewayErrorKind.Auth); RaiseAuthenticationFailed($"device signature rejected — {message}"); RaiseStatusChanged(ConnectionStatus.Error); return; @@ -2180,12 +2200,29 @@ private void HandleRequestError(string? method, JsonElement root) return; } - // Permanent auth failures — stop retrying and notify the app + // Permanent auth failures — stop retrying and notify the app. Check BOTH the top-level + // error.code and the structured error.details.code so a device-token mismatch delivered in + // either place is recognized (the gateway may send the reason only as a code with a generic + // message). var detailCode2 = TryGetErrorDetailCode(root); - if (method == "connect" && (IsTerminalAuthError(message) || IsTerminalAuthDetailCode(detailCode2))) + var topLevelCode = TryGetErrorTopLevelCode(root); + if (method == "connect" && + (IsTerminalAuthError(message) || IsTerminalAuthDetailCode(detailCode2) || IsTerminalAuthDetailCode(topLevelCode))) { _authFailed = true; - RaiseAuthenticationFailed(message); + var failureKind = GatewayErrorClassifier.ClassifyWithCode(message, topLevelCode, detailCode2); + RaiseConnectionFailure(failureKind); + // Preserve a structured device-token mismatch in the raised string so the connection + // manager can distinguish it (auto-recoverable) from a wrong shared token and other + // terminal auth failures. Only the device-token code is enriched; other codes pass + // through unchanged so they are never mistaken for a recoverable device-token drift. + var isDeviceMismatch = + string.Equals(detailCode2, GatewayErrorClassifier.DeviceTokenMismatchCode, StringComparison.OrdinalIgnoreCase) || + string.Equals(topLevelCode, GatewayErrorClassifier.DeviceTokenMismatchCode, StringComparison.OrdinalIgnoreCase); + var authMessage = isDeviceMismatch + ? $"{GatewayErrorClassifier.DeviceTokenMismatchCode}: {message}" + : message; + RaiseAuthenticationFailed(authMessage); RaiseStatusChanged(ConnectionStatus.Error); return; } @@ -2336,6 +2373,17 @@ private static string RedactAuthPayload(string authJson) return null; } + // Top-level error.code (distinct from the nested error.details.code). A gateway may deliver a + // terminal-auth reason (e.g. AUTH_DEVICE_TOKEN_MISMATCH) here with a generic message. + private static string? TryGetErrorTopLevelCode(JsonElement root) + { + if (!root.TryGetProperty("error", out var error) || error.ValueKind != JsonValueKind.Object) + return null; + if (error.TryGetProperty("code", out var code) && code.ValueKind == JsonValueKind.String) + return code.GetString(); + return null; + } + private static PairingConnectErrorDetails TryGetPairingConnectErrorDetails(JsonElement root) { if (!root.TryGetProperty("error", out var error) || error.ValueKind != JsonValueKind.Object) diff --git a/src/OpenClaw.Shared/SettingsData.cs b/src/OpenClaw.Shared/SettingsData.cs index cc5d4a997..f37a87c96 100644 --- a/src/OpenClaw.Shared/SettingsData.cs +++ b/src/OpenClaw.Shared/SettingsData.cs @@ -106,6 +106,13 @@ public record class SettingsData /// Run the local MCP HTTP server. Independent of EnableNodeMode. public bool EnableMcpServer { get; set; } = false; /// + /// Automatically self-repair an app-owned local WSL gateway when it becomes unreachable: + /// probe it and, if the gateway process is down, restart the managed WSL distro, re-arm the + /// keepalive, and reconnect — without user action. Strictly limited to setup-managed local + /// WSL gateways (never SSH/remote). Kill switch: set false to disable automatic repair. + /// + public bool EnableManagedLocalGatewayAutoRepair { get; set; } = true; + /// /// Hostnames the A2UI image renderer is allowed to fetch over HTTPS. /// Empty by default — agents can still ship inline data: images. Add hosts /// (e.g., "cdn.example.com") via the Settings window. diff --git a/src/OpenClaw.Shared/WebSocketClientBase.cs b/src/OpenClaw.Shared/WebSocketClientBase.cs index 3309764e5..5fa65acc3 100644 --- a/src/OpenClaw.Shared/WebSocketClientBase.cs +++ b/src/OpenClaw.Shared/WebSocketClientBase.cs @@ -7,6 +7,14 @@ namespace OpenClaw.Shared; +public readonly record struct ReconnectAuthorizationResult( + bool Allowed, + GatewayErrorKind FailureKind = GatewayErrorKind.Unknown, + string? Detail = null) +{ + public static ReconnectAuthorizationResult AllowedResult { get; } = new(true); +} + /// /// Abstract base class for WebSocket-based gateway clients. /// Extracts shared connection lifecycle: connect, listen, reconnect, send, dispose. @@ -43,6 +51,13 @@ public abstract class WebSocketClientBase : IDisposable // Events public event EventHandler? StatusChanged; public event EventHandler? AuthenticationFailed; + public event EventHandler? Disposed; + /// + /// Optional fail-closed authorization invoked immediately before every client-owned reconnect. + /// Managed-local callers use it to re-check endpoint provenance and explicit user intent. + /// + public Func>? + ReconnectAuthorizationAsync { get; set; } /// Reset reconnect backoff counter. Call after successful application-level handshake. protected void ResetReconnectAttempts() => _reconnectAttempts = 0; @@ -189,6 +204,7 @@ public async Task ConnectAsync() DisposeStaleSocket(ws); } _logger.Error($"{ClientRole} connection failed", ex); + OnConnectionException(ex); RaiseStatusChanged(ConnectionStatus.Error); if (!_disposed && !_cts.Token.IsCancellationRequested && ShouldAutoReconnect()) @@ -198,6 +214,14 @@ public async Task ConnectAsync() } } + /// + /// Lets a concrete client preserve a typed transport failure before the generic status event is + /// raised. The base class deliberately does not expose exception text to consumers. + /// + protected virtual void OnConnectionException(Exception exception) + { + } + private bool IsCurrentConnection(ClientWebSocket ws, long generation) => !_disposed && Interlocked.Read(ref _connectionGeneration) == generation @@ -376,6 +400,21 @@ protected async Task ReconnectWithBackoffAsync( break; } + if (ReconnectAuthorizationAsync is not null) + { + var authorization = + await ReconnectAuthorizationAsync(_cts.Token).ConfigureAwait(false); + if (!authorization.Allowed) + { + _logger.Warn( + $"{ClientRole} reconnect blocked by endpoint authorization policy: " + + (authorization.Detail ?? authorization.FailureKind.ToString())); + OnReconnectAuthorizationDenied(authorization); + RaiseStatusChanged(ConnectionStatus.Error); + break; + } + } + // Safely dispose old socket var oldSocket = expectedSocket ?? _webSocket; if (oldSocket != null) @@ -412,6 +451,11 @@ protected async Task ReconnectWithBackoffAsync( } } + protected virtual void OnReconnectAuthorizationDenied( + ReconnectAuthorizationResult authorization) + { + } + private bool IsReconnectOwner(ClientWebSocket? expectedSocket, long expectedGeneration) { if (expectedSocket is null || IsCurrentConnection(expectedSocket, expectedGeneration)) @@ -553,5 +597,7 @@ public void Dispose() // Don't dispose _cts immediately — listen loop or reconnect may still reference it. // It will be GC'd after all pending tasks complete. + try { Disposed?.Invoke(this, EventArgs.Empty); } + catch (Exception ex) { _logger.Debug($"{ClientRole} Disposed handler threw: {ex.Message}"); } } } diff --git a/src/OpenClaw.Shared/WindowsNodeClient.cs b/src/OpenClaw.Shared/WindowsNodeClient.cs index f20fe1407..b9cbcb475 100644 --- a/src/OpenClaw.Shared/WindowsNodeClient.cs +++ b/src/OpenClaw.Shared/WindowsNodeClient.cs @@ -79,6 +79,12 @@ public class WindowsNodeClient : WebSocketClientBase public event EventHandler? TransportConnected; /// Raised with a finite classification before a terminal handshake error is published. public event EventHandler? ConnectionFailure; + + protected override void OnReconnectAuthorizationDenied( + ReconnectAuthorizationResult authorization) + { + ConnectionFailure?.Invoke(this, authorization.FailureKind); + } public new bool IsConnected => _isConnected; public string? NodeId => _nodeId; @@ -861,6 +867,7 @@ private void HandleRequestError(JsonElement root) { var error = "Unknown error"; var errorCode = "none"; + string? detailsCode = null; string? pairingReason = null; string? pairingRequestId = null; @@ -880,6 +887,10 @@ private void HandleRequestError(JsonElement root) { pairingReason = reason; } + if (TryGetString(detailsProp, "code", out var dc)) + { + detailsCode = dc; + } pairingRequestId = TryGetSafePairingRequestId(detailsProp); } } @@ -912,15 +923,32 @@ private void HandleRequestError(JsonElement root) return; } + // Stale/rotated node DEVICE token — terminal, but the connection manager can + // self-recover by clearing only the node device token and reconnecting with a + // still-valid shared/bootstrap credential. Detected from the structured code + // (top-level or details.code) or explicit device phrasing, independent of generic + // message wording. _rateLimited stops this client's own auto-reconnect so the + // manager owns the single recovery reconnect (no dueling loops). + if (ClassifyConnectionFailure(error, errorCode, detailsCode) == GatewayErrorKind.DeviceTokenMismatch) + { + _rateLimited = true; + _logger.Warn($"[NODE] Node device token mismatch; stopping reconnect for recovery. Error: {TokenSanitizer.Sanitize(error)}"); + ConnectionFailure?.Invoke(this, GatewayErrorKind.DeviceTokenMismatch); + RaiseStatusChanged(ConnectionStatus.Error); + return; + } + // Rate-limit / terminal auth errors — stop reconnecting if (error.Contains("too many failed", StringComparison.OrdinalIgnoreCase) || error.Contains("rate limit", StringComparison.OrdinalIgnoreCase) || error.Contains("origin not allowed", StringComparison.OrdinalIgnoreCase) || - error.Contains("token mismatch", StringComparison.OrdinalIgnoreCase)) + error.Contains("token mismatch", StringComparison.OrdinalIgnoreCase) || + IsTerminalAuthCode(errorCode) || + IsTerminalAuthCode(detailsCode)) { _rateLimited = true; _logger.Warn($"[NODE] Terminal auth error; stopping reconnect. Error: {TokenSanitizer.Sanitize(error)}"); - ConnectionFailure?.Invoke(this, ClassifyConnectionFailure(error, errorCode)); + ConnectionFailure?.Invoke(this, ClassifyConnectionFailure(error, errorCode, detailsCode)); RaiseStatusChanged(ConnectionStatus.Error); return; } @@ -937,20 +965,27 @@ private void HandleRequestError(JsonElement root) } _logger.Error($"Node registration failed: {TokenSanitizer.Sanitize(error)} (code: {errorCode})"); - ConnectionFailure?.Invoke(this, ClassifyConnectionFailure(error, errorCode)); + ConnectionFailure?.Invoke(this, ClassifyConnectionFailure(error, errorCode, detailsCode)); RaiseStatusChanged(ConnectionStatus.Error); } - private static GatewayErrorKind ClassifyConnectionFailure(string error, string errorCode) + private static GatewayErrorKind ClassifyConnectionFailure(string error, string errorCode, string? detailsCode = null) { if (error.Contains("too many failed", StringComparison.OrdinalIgnoreCase)) return GatewayErrorKind.RateLimited; if (error.Contains("origin not allowed", StringComparison.OrdinalIgnoreCase)) return GatewayErrorKind.Auth; - return GatewayErrorClassifier.Classify($"{errorCode} {error}"); + return GatewayErrorClassifier.ClassifyWithCode(error, errorCode, detailsCode); } + // Structured terminal-auth codes (wrong shared/bootstrap token, rate limit, token not + // configured, device-token mismatch). These are permanent for the current connection, so the + // node client must stop its own auto-reconnect even when the human message is generic. + private static bool IsTerminalAuthCode(string? code) => code is + "AUTH_TOKEN_MISMATCH" or "AUTH_BOOTSTRAP_TOKEN_INVALID" or + "AUTH_DEVICE_TOKEN_MISMATCH" or "AUTH_RATE_LIMITED" or "AUTH_TOKEN_NOT_CONFIGURED"; + private bool PayloadTargetsCurrentDevice(JsonElement payload) { if (TryGetString(payload, "deviceId", out var deviceId) && diff --git a/src/OpenClaw.Shared/WindowsStartupTaskRegistration.cs b/src/OpenClaw.Shared/WindowsStartupTaskRegistration.cs index 2a24ce843..2769d1cb0 100644 --- a/src/OpenClaw.Shared/WindowsStartupTaskRegistration.cs +++ b/src/OpenClaw.Shared/WindowsStartupTaskRegistration.cs @@ -74,7 +74,7 @@ private static bool Run(ProcessStartInfo startInfo) } } - internal static string ResolveSchtasksPath() + public static string ResolveSchtasksPath() { var systemRoot = Environment.GetFolderPath(Environment.SpecialFolder.Windows); if (string.IsNullOrWhiteSpace(systemRoot)) diff --git a/src/OpenClaw.Tray.WinUI/App.xaml.cs b/src/OpenClaw.Tray.WinUI/App.xaml.cs index 2f7f1707b..ed29a8a2e 100644 --- a/src/OpenClaw.Tray.WinUI/App.xaml.cs +++ b/src/OpenClaw.Tray.WinUI/App.xaml.cs @@ -52,6 +52,8 @@ public partial class App : Application, OpenClawTray.Services.IAppCommands private TrayIconCoordinator? _trayIconCoordinator; private GatewayConnectionManager? _connectionManager; private GatewayRegistry? _gatewayRegistry; + private OpenClawTray.Services.ManagedLocalGatewayAutoRepairMonitor? _managedLocalAutoRepairMonitor; + private ManagedLocalGatewayPortProvenanceService? _managedLocalPortProvenance; private OpenClawTray.Chat.OpenClawChatCoordinator? _chatCoordinator; /// @@ -92,6 +94,8 @@ public partial class App : Application, OpenClawTray.Services.IAppCommands public IOperatorGatewayClient? GatewayClient => _connectionManager?.OperatorClient; public GatewayRegistry? Registry => _gatewayRegistry; public GatewayConnectionManager? ConnectionManager => _connectionManager; + internal ManagedLocalGatewayPortProvenanceService? ManagedLocalPortProvenance => + _managedLocalPortProvenance; internal SettingsManager Settings => _settings ?? throw new InvalidOperationException("Settings are not initialized."); internal SettingsManager? SettingsOrNull => _settings; internal string DataDirectoryPath => DataPath; @@ -776,13 +780,16 @@ _dispatcherQueue is null } }; // SshTunnelService implements ISshTunnelManager directly — no shim needed + var managedLocalPortProvenance = _managedLocalPortProvenance = + new ManagedLocalGatewayPortProvenanceService(appLogger); _connectionManager = new GatewayConnectionManager( credentialResolver, clientFactory, _gatewayRegistry, appLogger, identityStore: new DeviceIdentityFileStore(appLogger), nodeConnector: nodeConnector, isNodeEnabled: IsGatewayNodeEnabled, diagnostics: diagnostics, - tunnelManager: _sshTunnelService); + tunnelManager: _sshTunnelService, + endpointProvenanceProbe: managedLocalPortProvenance.InspectAsync); _connectionManager.OperatorClientChanged += OnOperatorClientChanged; _connectionManager.StateChanged += OnManagerStateChanged; @@ -825,6 +832,57 @@ _dispatcherQueue is null // runs detached from the tray — see WslDistroKeepAlive in LocalGatewaySetup.cs. var wslKeepAlive = new WslGatewayKeepAliveService(() => _settings, () => _gatewayRegistry); _ = Task.Run(wslKeepAlive.TryEnsureAsync); + + // Automatic self-repair for app-owned setup-managed local WSL gateways: if the local + // gateway process goes down, probe it and (only if actually unreachable) restart the WSL + // distro, re-arm the keepalive, and reconnect — without user action. Strictly gated to + // setup-managed local WSL gateways; the reconnect is gateway-pinned + cancellable so a + // gateway switch or shutdown mid-repair cannot disrupt another gateway. Kill switch: + // Settings.EnableManagedLocalGatewayAutoRepair. + var managedLocalRestarter = new OpenClawTray.Services.WslManagedLocalGatewayRestarter( + new WslGatewayController(new WslExeCommandRunner(new AppLogger(), defaultTimeout: TimeSpan.FromSeconds(30)), appLogger)); + var managedLocalRepairCoordinator = new OpenClawTray.Services.ManagedLocalGatewayRepairCoordinator( + _gatewayRegistry, + managedLocalRestarter, + (url, ct) => OpenClawTray.Services.GatewayReachabilityProbe.IsReachableAsync(url, ct), + (gatewayId, ct) => _connectionManager?.ReconnectIfCurrentAsync(gatewayId, ct) ?? Task.FromResult(false), + () => _connectionManager?.CurrentSnapshot.OperatorState == RoleConnectionState.Connected, + _ => wslKeepAlive.TryEnsureAsync(), + diagnostics, + appLogger, + tryAcquireLifecycleLease: () => _connectionManager?.TryAcquireGatewayLifecycleLease(), + isRestartStillWarranted: () => OpenClawTray.Services.ManagedLocalGatewayAutoRepairMonitor.IsRepairCandidate( + _connectionManager?.CurrentSnapshot ?? GatewayConnectionSnapshot.Idle), + isAutomaticRepairAllowed: gatewayId => _connectionManager?.IsAutomaticReconnectAllowed(gatewayId) ?? false, + repairPortConflictAsync: (record, ct) => + OpenClawTray.Services.ManagedLocalGatewayAutoRepairMonitor.IsRepairCandidate( + _connectionManager?.CurrentSnapshot ?? GatewayConnectionSnapshot.Idle) && + _connectionManager?.CurrentSnapshot.OperatorErrorKind == GatewayErrorKind.LocalPortConflict + ? managedLocalPortProvenance.RepairConflictAsync( + record, + ct, + canContinue: () => + string.Equals( + _gatewayRegistry?.ActiveGatewayId, + record.Id, + StringComparison.Ordinal) && + (_connectionManager?.IsAutomaticReconnectAllowed(record.Id) ?? false)) + : Task.FromResult(new ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome.NotNeeded)), + isPortConflictCandidate: () => + _connectionManager?.CurrentSnapshot.OperatorErrorKind == GatewayErrorKind.LocalPortConflict); + _managedLocalAutoRepairMonitor = new OpenClawTray.Services.ManagedLocalGatewayAutoRepairMonitor( + () => _connectionManager?.CurrentSnapshot ?? GatewayConnectionSnapshot.Idle, + _gatewayRegistry, + ct => managedLocalRepairCoordinator.TryRepairActiveGatewayAsync(ct), + id => managedLocalRepairCoordinator.ResetAttemptBudget(id), + () => (_settings?.EnableManagedLocalGatewayAutoRepair ?? true) + && !(_connectionManager?.IsManualGatewayLifecycleInProgress ?? false), + diagnostics, + appLogger, + isAutomaticRepairAllowed: gatewayId => _connectionManager?.IsAutomaticReconnectAllowed(gatewayId) ?? false); + _managedLocalAutoRepairMonitor.Start(); + InitializeGatewayClient(); // Pre-warm chat window (WebView2 init takes 1-3s, do it now so left-click is instant) @@ -1140,7 +1198,7 @@ private void OnTrayMenuItemClicked(object? sender, string action) case "status": ShowStatusDetail(); break; case "reconnect": ReconnectWithSyncedBrowserProxyForward(); break; case "disconnect": - _ = _connectionManager?.DisconnectAsync(); + _ = _connectionManager?.DisconnectByUserAsync(); LocalDisconnectCleanup(); break; case "connection": ShowHub("connection"); break; @@ -2098,7 +2156,29 @@ private void OnManagerStateChanged(object? sender, GatewayConnectionSnapshot sna sharedGatewayTokenResolver: () => _gatewayRegistry?.GetActive()?.SharedGatewayToken, browserControlPortResolver: () => _gatewayRegistry?.GetActive()?.BrowserControlPort, activeGatewayTunnelResolver: () => _gatewayRegistry?.GetActive()?.SshTunnel, - activeGatewayUrlResolver: () => _gatewayRegistry?.GetActive()?.Url); + activeGatewayUrlResolver: () => _gatewayRegistry?.GetActive()?.Url, + browserControlAuthorization: async (uri, cancellationToken) => + { + var record = _gatewayRegistry?.GetActive(); + if (record is null || !uri.IsLoopback) + return false; + if (record.SshTunnel is not null) + return _sshTunnelService?.IsActive == true; + if (_managedLocalPortProvenance is null || + GatewayRecordEditing.ResolveManagedDistroName(record) is null) + { + return false; + } + + var controlRecord = record with + { + Url = $"ws://localhost:{uri.Port}", + IsLocal = true, + }; + return (await _managedLocalPortProvenance.InspectAsync( + controlRecord, + cancellationToken)).Kind == GatewayEndpointProvenanceKind.ExpectedManagedGateway; + }); _nodeService.StatusChanged += OnNodeStatusChanged; _nodeService.NotificationRequested += OnNodeNotificationRequested; _nodeService.ToastRequested += OnNodeToastRequested; @@ -3386,7 +3466,7 @@ internal void ShowHub(string? navigateTo = null, bool activate = true) }; _hubWindow.DisconnectAction = () => { - _ = _connectionManager?.DisconnectAsync(); + _ = _connectionManager?.DisconnectByUserAsync(); // Status is updated by OnManagerStateChanged when disconnect completes. UpdateTrayIcon(); }; @@ -4093,6 +4173,8 @@ private bool TryResolveChatCredentials( _settings.GetEffectiveGatewayUrl(), _settings.LegacyToken, _settings.LegacyBootstrapToken, + (record, candidate) => + _managedLocalPortProvenance?.IsStrongCredentialAllowed(record, candidate) == true, out var credential) || credential == null) { @@ -4150,7 +4232,7 @@ private void OpenDashboard(string? path = null) void IAppCommands.Reconnect() => ReconnectWithSyncedBrowserProxyForward(); void IAppCommands.Disconnect() { - _ = _connectionManager?.DisconnectAsync(); + _ = _connectionManager?.DisconnectByUserAsync(); UpdateTrayIcon(); } void IAppCommands.ShowVoiceOverlay() => ShowHub("voice"); @@ -4639,7 +4721,18 @@ private async Task ExitApplicationAsync() _chatCoordinator = null; }); - // Dispose runtime services + // Dispose runtime services. Stop the auto-repair monitor BEFORE the connection manager so an + // in-flight repair cannot drive a reconnect into a disposing manager. + var autoRepairMonitor = _managedLocalAutoRepairMonitor; + if (autoRepairMonitor != null) + { + await SafeShutdownStepAsync("managed-local auto-repair monitor", async () => + { + await autoRepairMonitor.DisposeAsync(); + }); + _managedLocalAutoRepairMonitor = null; + } + var connectionManager = _connectionManager; if (connectionManager != null) { diff --git a/src/OpenClaw.Tray.WinUI/Pages/ChatPage.xaml.cs b/src/OpenClaw.Tray.WinUI/Pages/ChatPage.xaml.cs index 17e6447e5..212d149f4 100644 --- a/src/OpenClaw.Tray.WinUI/Pages/ChatPage.xaml.cs +++ b/src/OpenClaw.Tray.WinUI/Pages/ChatPage.xaml.cs @@ -212,6 +212,9 @@ private void ApplyChatSurface() settings.GetEffectiveGatewayUrl(), settings.LegacyToken, settings.LegacyBootstrapToken, + (record, candidate) => + (App.Current as App)?.ManagedLocalPortProvenance + ?.IsStrongCredentialAllowed(record, candidate) == true, out var credential) && credential is { IsBootstrapToken: false } ? ChatSurfaceResolver.BuildChatUrl(credential.GatewayUrl, credential.Token) @@ -569,6 +572,9 @@ private async Task InitializeWebViewAsync(SettingsManager settings) settings.GetEffectiveGatewayUrl(), settings.LegacyToken, settings.LegacyBootstrapToken, + (record, candidate) => + CurrentApp.ManagedLocalPortProvenance + ?.IsStrongCredentialAllowed(record, candidate) == true, out var credential) || credential == null) { diff --git a/src/OpenClaw.Tray.WinUI/Pages/ConnectionPage.xaml.cs b/src/OpenClaw.Tray.WinUI/Pages/ConnectionPage.xaml.cs index e9df3ca28..08cecc18b 100644 --- a/src/OpenClaw.Tray.WinUI/Pages/ConnectionPage.xaml.cs +++ b/src/OpenClaw.Tray.WinUI/Pages/ConnectionPage.xaml.cs @@ -1077,6 +1077,7 @@ private void ApplyRecoveryBody(ConnectionPagePlan plan) RecoveryCategory.Tls => LocalizationHelper.GetString("ConnectionPage_RecoveryHeaderTls"), RecoveryCategory.RateLimited => LocalizationHelper.GetString("ConnectionPage_RecoveryHeaderRateLimited"), RecoveryCategory.Tailscale => "Check Tailscale access", + RecoveryCategory.LocalPortConflict => "Resolve the local gateway port conflict:", _ => LocalizationHelper.GetString("ConnectionPage_RecoveryHeaderServer"), }; @@ -1129,6 +1130,12 @@ private void ApplyRecoveryBody(ConnectionPagePlan plan) "Confirm this PC and the generated WSL gateway belong to the same tailnet.", "Open the managed WSL gateway terminal as root and check tailscaled, Tailscale Serve, and the OpenClaw gateway service. Funnel is unsupported; remove any Funnel route. Companion keeps using WSS and never falls back to localhost.", }, + RecoveryCategory.LocalPortConflict => new[] + { + "Another process is listening on the managed WSL gateway's local address.", + "OpenClaw automatically removes only a fully verified obsolete OpenClaw gateway. Unknown processes are never stopped.", + "Stop the conflicting app or run Reconfigure… to choose a different gateway address, then retry.", + }, _ => new[] { LocalizationHelper.GetString("ConnectionPage_RecoveryDefaultBullet1"), @@ -2034,7 +2041,7 @@ private void OnStripPrimaryClicked(object sender, RoutedEventArgs e) ((IAppCommands)CurrentApp).Reconnect(); break; case ConnectionPrimaryAction.Cancel: - _ = _connectionManager?.DisconnectAsync(); + _ = _connectionManager?.DisconnectByUserAsync(); break; case ConnectionPrimaryAction.RestartTunnel: OnRestartTunnel(sender, e); @@ -2116,6 +2123,11 @@ private async Task RunWslGatewayControlAsync(WslGatewayControlAction action) return; } + // Stop is explicit user intent. Record it before waiting for the lifecycle lease so an + // in-flight automatic repair cannot reconnect/restart this gateway while Stop is queued. + if (action == WslGatewayControlAction.Stop && activeRecord is not null) + _connectionManager?.SetGatewayConnectionIntent(activeRecord.Id, shouldBeConnected: false); + var cts = new CancellationTokenSource(); _gatewayHostActionCts = cts; var cancellationToken = cts.Token; @@ -2124,8 +2136,39 @@ private async Task RunWslGatewayControlAsync(WslGatewayControlAction action) var verb = WslGatewayControlCommandBuilder.ToVerb(action); SetGatewayHostActionStatus($"{ActionInProgressLabel(action)} gateway in {accessPlan.DistroName}…"); + // Suppress managed-local auto-repair from STARTING a new repair while this manual action runs, + // so the two paths don't kick off concurrent distro restarts (an already-in-flight repair is + // single-flighted and re-checks the active gateway, so it self-reconciles). Acquired as the + // first statement inside the try so the finally always disposes it — a throw before this point + // cannot leak the suppression and permanently wedge self-healing. + IDisposable? manualLifecycleScope = null; + try { + manualLifecycleScope = _connectionManager is null + ? null + : await _connectionManager.BeginManualGatewayLifecycleOperationAsync(cancellationToken); + + // The lease await above can block for seconds while an auto-repair distro restart holds it. + // If the user switched OR edited the active gateway during that wait, this action's captured + // target (activeRecord/accessPlan) is stale. Re-read and RE-CLASSIFY the active gateway and + // require the same id, WSL controllability, and resolved distro as when we started — + // otherwise a Stop/Restart could disconnect the switched-to gateway or operate on a stale + // distro after a same-id edit repointed the record. + var activeNow = _gatewayRegistry?.GetActive(); + var planNow = GatewayHostAccessClassifier.Classify(activeNow); + if (activeRecord is null || activeNow is null || + !string.Equals(activeNow.Id, activeRecord.Id, StringComparison.Ordinal) || + !planNow.CanControlWslGateway || + !string.Equals(planNow.DistroName, accessPlan.DistroName, StringComparison.OrdinalIgnoreCase)) + { + SetGatewayHostActionStatus("Active gateway changed; cancelled this action."); + return; + } + + if (action is WslGatewayControlAction.Start or WslGatewayControlAction.Restart) + _connectionManager?.SetGatewayConnectionIntent(activeRecord.Id, shouldBeConnected: true); + if (action == WslGatewayControlAction.Stop && _connectionManager != null) { try @@ -2157,9 +2200,29 @@ private async Task RunWslGatewayControlAsync(WslGatewayControlAction action) return; } + var activeAfterAction = _gatewayRegistry?.GetActive(); + var planAfterAction = GatewayHostAccessClassifier.Classify(activeAfterAction); + if (activeRecord is null || activeAfterAction is null || + !string.Equals(activeAfterAction.Id, activeRecord.Id, StringComparison.Ordinal) || + !planAfterAction.CanControlWslGateway || + !string.Equals( + planAfterAction.DistroName, + accessPlan.DistroName, + StringComparison.OrdinalIgnoreCase)) + { + SetGatewayHostActionStatus($"Gateway {PastTense(action)}. Active gateway changed; not reconnecting."); + return; + } + + if (_connectionManager is null || + !await _connectionManager.ReconnectIfCurrentAsync(activeRecord.Id, cancellationToken)) + { + SetGatewayHostActionStatus($"Gateway {PastTense(action)}. Reconnect skipped because the gateway was switched or disconnected."); + return; + } + SetGatewayHostActionStatus($"Gateway {PastTense(action)}. Reconnecting…"); BeginReconnectMask(); - ((IAppCommands)CurrentApp).Reconnect(); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -2173,6 +2236,7 @@ private async Task RunWslGatewayControlAsync(WslGatewayControlAction action) finally { _gatewayHostActionInProgress = false; + manualLifecycleScope?.Dispose(); if (ReferenceEquals(_gatewayHostActionCts, cts)) { _gatewayHostActionCts = null; @@ -2401,19 +2465,43 @@ private async Task OnConnectSavedGatewayAsync(object sender) } } - private void OnSavedRowOpenDashboard(object sender, RoutedEventArgs e) + private void OnSavedRowOpenDashboard(object sender, RoutedEventArgs e) => + AsyncEventHandlerGuard.Run( + () => OnSavedRowOpenDashboardAsync(sender), + new AppLogger(), + nameof(OnSavedRowOpenDashboard)); + + private async Task OnSavedRowOpenDashboardAsync(object sender) { if (sender is not MenuFlyoutItem item || item.Tag is not string gwId) return; var rec = _gatewayRegistry?.GetById(gwId); if (rec == null) return; try { + if (!string.IsNullOrWhiteSpace(rec.SharedGatewayToken)) + { + var provenanceService = CurrentApp.ManagedLocalPortProvenance; + if (provenanceService is null) + return; + _ = await provenanceService.InspectAsync(rec); + var candidate = new GatewayCredential( + rec.SharedGatewayToken!, + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken); + if (!provenanceService.IsStrongCredentialAllowed(rec, candidate)) + { + CurrentApp.ShowTransientConnectionError( + "Dashboard blocked because the saved gateway address is not owned by the verified managed gateway."); + return; + } + } + var url = GatewayDashboardUrlBuilder.Build( rec.Url, path: null, rec.SharedGatewayToken, appendSharedGatewayToken: !string.IsNullOrWhiteSpace(rec.SharedGatewayToken)); - _ = global::Windows.System.Launcher.LaunchUriAsync(new Uri(url)); + await global::Windows.System.Launcher.LaunchUriAsync(new Uri(url)); } catch (Exception ex) { diff --git a/src/OpenClaw.Tray.WinUI/Pages/ConnectionPagePlan.cs b/src/OpenClaw.Tray.WinUI/Pages/ConnectionPagePlan.cs index 8d5feee29..fe756e373 100644 --- a/src/OpenClaw.Tray.WinUI/Pages/ConnectionPagePlan.cs +++ b/src/OpenClaw.Tray.WinUI/Pages/ConnectionPagePlan.cs @@ -119,6 +119,8 @@ internal enum RecoveryCategory RateLimited, /// A managed Tailscale endpoint cannot be reached from this Companion. Tailscale, + /// A different or unverified local process owns the managed gateway port. + LocalPortConflict, } /// @@ -442,7 +444,7 @@ private static ConnectionPagePlan BuildRecoveryFromError( { var errRaw = snap.OperatorError ?? snap.NodeError ?? ""; var err = ConnectionCardPlanSanitizer.Sanitize(errRaw); - var category = ClassifyError(err); + var category = ClassifyError(snap, err); var url = ConnectionCardPlanSanitizer.SanitizeGatewayUrl(rec?.Url ?? snap.GatewayUrl); if (category == RecoveryCategory.Network && @@ -470,6 +472,22 @@ private static ConnectionPagePlan BuildRecoveryFromError( return category switch { + RecoveryCategory.LocalPortConflict => new ConnectionPagePlan + { + Mode = ConnectionPageMode.Recovery, + Recovery = RecoveryCategory.LocalPortConflict, + StripGlyph = OpenClawTray.Helpers.FluentIconCatalog.StatusWarn, + StripAccent = ConnectionAccent.Caution, + StripHeadline = "Local gateway port conflict", + StripSub = "Another process is using this managed gateway address. OpenClaw will not send gateway credentials to an unverified listener.", + StripPrimaryLabel = "Retry", + StripPrimaryAction = ConnectionPrimaryAction.Retry, + ActiveGatewayDisplayName = name, + ActiveGatewayDetailLine = url, + ActiveGatewayHasSshTunnel = false, + RelevantGatewayId = rec?.Id, + }, + RecoveryCategory.Auth => new ConnectionPagePlan { Mode = ConnectionPageMode.Recovery, @@ -861,17 +879,20 @@ private static int CountEnabledCapabilities(SettingsManager s) return n; } - private static RecoveryCategory ClassifyError(string err) + private static RecoveryCategory ClassifyError(GatewayConnectionSnapshot snapshot, string err) { // Delegate the heuristic matching to the pure, unit-tested Shared // classifier so the same kinds drive both setup and recovery copy. - return OpenClaw.Shared.GatewayErrorClassifier.Classify(err) switch + return (snapshot.OperatorErrorKind ?? + OpenClaw.Shared.GatewayErrorClassifier.Classify(err)) switch { OpenClaw.Shared.GatewayErrorKind.ScopeMismatch => RecoveryCategory.Scope, OpenClaw.Shared.GatewayErrorKind.TokenDrift => RecoveryCategory.TokenDrift, + OpenClaw.Shared.GatewayErrorKind.DeviceTokenMismatch => RecoveryCategory.TokenDrift, OpenClaw.Shared.GatewayErrorKind.Auth => RecoveryCategory.Auth, OpenClaw.Shared.GatewayErrorKind.Tls => RecoveryCategory.Tls, OpenClaw.Shared.GatewayErrorKind.Tunnel => RecoveryCategory.Tunnel, + OpenClaw.Shared.GatewayErrorKind.LocalPortConflict => RecoveryCategory.LocalPortConflict, OpenClaw.Shared.GatewayErrorKind.Server => RecoveryCategory.Server, OpenClaw.Shared.GatewayErrorKind.RateLimited => RecoveryCategory.RateLimited, OpenClaw.Shared.GatewayErrorKind.PairingRejected => RecoveryCategory.Auth, diff --git a/src/OpenClaw.Tray.WinUI/Services/ManagedLocalGatewayAutoRepairMonitor.cs b/src/OpenClaw.Tray.WinUI/Services/ManagedLocalGatewayAutoRepairMonitor.cs new file mode 100644 index 000000000..60e347962 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Services/ManagedLocalGatewayAutoRepairMonitor.cs @@ -0,0 +1,298 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using OpenClaw.Connection; +using OpenClaw.Shared; + +namespace OpenClawTray.Services; + +/// +/// Watches the operator connection and, when an app-owned setup-managed local WSL gateway looks +/// transport-unreachable for a sustained period, invokes +/// to (probe and, if down) restart the WSL +/// distro and reconnect — so a crashed/stopped local gateway self-heals without user action. +/// +/// +/// Safety design (addresses the multi-model review findings): +/// +/// Only acts on gateways. +/// Repairs only a transport failure. Auth/pairing/rate-limit/scope/TLS/tunnel and +/// token-drift are excluded via — a distro restart cannot +/// fix credentials, and an intentional user disconnect (Idle) is never repaired. +/// Honors a startup grace period so a slow WSL cold start is not interrupted, and an +/// unhealthy threshold + cooldown so it cannot restart-storm. +/// Kill switch: isEnabled (settings) can disable automatic repair entirely. +/// Delegates to the coordinator (single-flight, budgeted, probe-before-restart); owns no +/// reconnect loop of its own. +/// +/// +public sealed class ManagedLocalGatewayAutoRepairMonitor : IAsyncDisposable +{ + private readonly Func _getSnapshot; + private readonly GatewayRegistry _registry; + private readonly Func> _repairAsync; + private readonly Action _resetRepairBudget; + private readonly Func _isEnabled; + private readonly Func _isAutomaticRepairAllowed; + private readonly ConnectionDiagnostics _diagnostics; + private readonly IOpenClawLogger _logger; + private readonly IClock _clock; + private readonly TimeSpan _unhealthyThreshold; + private readonly TimeSpan _pollInterval; + private readonly TimeSpan _cooldown; + private readonly TimeSpan _startupGrace; + private readonly Func _delay; + + private readonly CancellationTokenSource _cts = new(); + private Task? _loop; + private bool _disposed; + private DateTime? _startedAt; + + private DateTime? _operatorUnhealthySince; + private string? _unhealthyGatewayId; + // Last repair attempt time keyed by gateway id, so switching away from and back to a gateway does + // not drop its cooldown (a single-slot timestamp would let alternating gateways restart-storm). + private readonly Dictionary _lastRepairByGateway = new(StringComparer.Ordinal); + + public ManagedLocalGatewayAutoRepairMonitor( + Func getSnapshot, + GatewayRegistry registry, + Func> repairAsync, + Action resetRepairBudget, + Func isEnabled, + ConnectionDiagnostics diagnostics, + IOpenClawLogger logger, + IClock? clock = null, + TimeSpan? unhealthyThreshold = null, + TimeSpan? pollInterval = null, + TimeSpan? cooldown = null, + TimeSpan? startupGrace = null, + Func? delay = null, + Func? isAutomaticRepairAllowed = null) + { + _getSnapshot = getSnapshot ?? throw new ArgumentNullException(nameof(getSnapshot)); + _registry = registry ?? throw new ArgumentNullException(nameof(registry)); + _repairAsync = repairAsync ?? throw new ArgumentNullException(nameof(repairAsync)); + _resetRepairBudget = resetRepairBudget ?? (static _ => { }); + _isEnabled = isEnabled ?? (static () => true); + _isAutomaticRepairAllowed = isAutomaticRepairAllowed ?? (static _ => true); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _clock = clock ?? SystemClock.Instance; + _unhealthyThreshold = unhealthyThreshold ?? TimeSpan.FromSeconds(15); + _pollInterval = pollInterval ?? TimeSpan.FromSeconds(5); + _cooldown = cooldown ?? TimeSpan.FromSeconds(90); + _startupGrace = startupGrace ?? TimeSpan.FromSeconds(45); + _delay = delay ?? Task.Delay; + _startedAt = _clock.UtcNow; + } + + /// Starts the background monitor loop. Idempotent. + public void Start() + { + if (_disposed || _loop != null) + return; + + _loop = Task.Run(() => RunAsync(_cts.Token)); + } + + private async Task RunAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + await EvaluateOnceAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + _logger.Warn($"[AutoRepair] Monitor evaluation failed: {ex.Message}"); + } + + try + { + await _delay(_pollInterval, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + break; + } + } + } + + /// One evaluation tick. Internal for unit tests. + internal async Task EvaluateOnceAsync(CancellationToken cancellationToken) + { + var record = _registry.GetActive(); + + // Only auto-repair app-owned setup-managed local WSL gateways. + if (record is null || !WslKeepAlivePolicy.IsSetupManagedLocalRecord(record)) + { + ClearUnhealthyTracking(); + return; + } + + if (!_isAutomaticRepairAllowed(record.Id)) + { + ClearUnhealthyTracking(); + return; + } + + // Kill switch. + if (!_isEnabled()) + { + ClearUnhealthyTracking(); + return; + } + + var snapshot = _getSnapshot(); + var now = _clock.UtcNow; + + // The snapshot can briefly describe a DIFFERENT gateway than the active record during a switch: + // SwitchGatewayAsync sets the active id BEFORE the connection state catches up, so a tick here + // could read gateway B as active while the snapshot still reports gateway A's Connected state. + // Attributing A's health to B would wrongly reset B's restart budget (letting alternating + // gateways bypass the bound); attributing A's outage to B could repair the wrong gateway. Skip + // until the snapshot's gateway matches the active record. + if (!string.IsNullOrEmpty(snapshot.GatewayId) && + !string.Equals(snapshot.GatewayId, record.Id, StringComparison.Ordinal)) + { + return; + } + + if (!IsRepairCandidate(snapshot)) + { + ClearUnhealthyTracking(); + // Genuine health (any route) frees the coordinator's restart budget FOR THIS GATEWAY so a + // later outage starts fresh instead of inheriting a count from an earlier repair. + if (snapshot.OperatorState == RoleConnectionState.Connected) + _resetRepairBudget(record.Id); + return; + } + + // Startup grace: never restart during the initial window, so a slow WSL cold start is not + // interrupted (a fresh Connecting/Error with no classified failure is normal at startup). + if (_startedAt is { } startedAt && now - startedAt < _startupGrace) + return; + + // Per-gateway unhealthy timer: a freshly-activated gateway accrues its own threshold. + if (!string.Equals(_unhealthyGatewayId, record.Id, StringComparison.Ordinal)) + { + _operatorUnhealthySince = now; + _unhealthyGatewayId = record.Id; + } + else + { + _operatorUnhealthySince ??= now; + } + + if (now - _operatorUnhealthySince.Value < _unhealthyThreshold) + return; + + if (_lastRepairByGateway.TryGetValue(record.Id, out var lastAttempt) && now - lastAttempt < _cooldown) + return; + + // Prune cooldown entries that have fully elapsed (they no longer gate anything) so the map + // stays bounded even across many create/delete gateway churns in a long-running session. + PruneExpiredCooldowns(now); + _lastRepairByGateway[record.Id] = now; + + var unhealthyMs = (now - _operatorUnhealthySince.Value).TotalMilliseconds; + _diagnostics.Record("setup", "Local gateway unreachable — attempting automatic repair", $"unhealthyForMs={unhealthyMs:0}"); + + var result = await _repairAsync(cancellationToken).ConfigureAwait(false); + _diagnostics.Record("setup", $"Automatic local gateway repair: {result.Outcome}", result.Detail); + + if (result.Succeeded) + ClearUnhealthyTracking(); + } + + private void ClearUnhealthyTracking() + { + _operatorUnhealthySince = null; + _unhealthyGatewayId = null; + } + + private void PruneExpiredCooldowns(DateTime now) + { + if (_lastRepairByGateway.Count == 0) + return; + + List? expired = null; + foreach (var kvp in _lastRepairByGateway) + { + if (now - kvp.Value >= _cooldown) + (expired ??= []).Add(kvp.Key); + } + + if (expired is null) + return; + + foreach (var id in expired) + _lastRepairByGateway.Remove(id); + } + + // Repair applies only to a genuine transport outage: the operator is Connecting/Error AND the + // classified failure (if any) is not an auth/pairing/credential/rate-limit/TLS/tunnel/scope or + // token-drift problem, none of which a distro restart can fix. A null error (Connecting with no + // classified reason) is treated as transport-ish; the coordinator's probe is the real gate for + // whether a restart actually happens. Public so the coordinator can re-check this same condition + // immediately before a restart (the failure may have become terminal-auth during the verify wait). + public static bool IsTransportUnreachable(GatewayConnectionSnapshot snapshot) + { + if (snapshot.OperatorState is not (RoleConnectionState.Connecting or RoleConnectionState.Error)) + return false; + + if (snapshot.OperatorPairingRequired || snapshot.OperatorCredentialBootstrapRequired) + return false; + + var kind = snapshot.OperatorErrorKind ?? + GatewayErrorClassifier.Classify(snapshot.OperatorError); + + // A cold start can sit in Connecting before any classified failure exists. That remains + // repairable after grace/threshold + the coordinator's real reachability probe. + if (snapshot.OperatorState == RoleConnectionState.Connecting && + string.IsNullOrWhiteSpace(snapshot.OperatorError) && + snapshot.OperatorErrorKind is null) + { + return true; + } + + return kind switch + { + GatewayErrorKind.Network or GatewayErrorKind.Server => true, + _ => false, // Unknown, Auth, DeviceTokenMismatch, TokenDrift, ScopeMismatch, Pairing*, RateLimited, Tls, Tunnel + }; + } + + public static bool IsRepairCandidate(GatewayConnectionSnapshot snapshot) => + IsTransportUnreachable(snapshot) || + (snapshot.OperatorState == RoleConnectionState.Error && + snapshot.OperatorErrorKind == GatewayErrorKind.LocalPortConflict); + + public async ValueTask DisposeAsync() + { + if (_disposed) + return; + + _disposed = true; + try { _cts.Cancel(); } + // slopwatch-ignore: SW003 Shutdown cancellation is expected; callers already preserve safe state. + catch (ObjectDisposedException) { } + + // Await the loop so an in-flight evaluation cannot drive a reconnect into a disposing + // manager, bounding the wait so shutdown cannot hang on a stuck repair. + if (_loop is { } loop) + { + try { await Task.WhenAny(loop, Task.Delay(TimeSpan.FromSeconds(3))).ConfigureAwait(false); } + catch (Exception ex) { _logger.Warn($"[AutoRepair] Dispose await failed: {ex.Message}"); } + } + + _cts.Dispose(); + } +} diff --git a/src/OpenClaw.Tray.WinUI/Services/ManagedLocalGatewayRepairAdapters.cs b/src/OpenClaw.Tray.WinUI/Services/ManagedLocalGatewayRepairAdapters.cs new file mode 100644 index 000000000..d55201480 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Services/ManagedLocalGatewayRepairAdapters.cs @@ -0,0 +1,79 @@ +using System; +using System.Net.Sockets; +using System.Threading; +using System.Threading.Tasks; +using OpenClaw.Connection; + +namespace OpenClawTray.Services; + +/// +/// backed by . +/// Restart output is not surfaced verbatim to avoid leaking command echo into logs; only +/// success/failure and a bounded, sanitized summary are propagated. +/// +public sealed class WslManagedLocalGatewayRestarter(WslGatewayController controller) : IManagedLocalGatewayRestarter +{ + private readonly WslGatewayController _controller = controller ?? throw new ArgumentNullException(nameof(controller)); + + public async Task RestartAsync(string distroName, CancellationToken cancellationToken) + { + var result = await _controller + .RunAsync(distroName, WslGatewayControlAction.Restart, cancellationToken) + .ConfigureAwait(false); + + if (result.Success) + return new ManagedLocalGatewayRestartResult(true); + + // The in-place restart failed — the distro/VM may be wedged and unable to run an in-distro + // command (an in-distro command also times out to a failure rather than hanging forever). + // Escalate to a host-side terminate + cold restart, which recovers a hung WSL instance + // (ForceRestartAsync logs the terminate). + result = await _controller + .ForceRestartAsync(distroName, cancellationToken) + .ConfigureAwait(false); + + if (result.Success) + return new ManagedLocalGatewayRestartResult(true); + + var detail = result.ExitCode == 0 + ? "Gateway restart reported failure." + : $"wsl.exe exited with code {result.ExitCode}."; + return new ManagedLocalGatewayRestartResult(false, detail); + } +} + +/// +/// Lightweight TCP reachability probe for a gateway URL — used by the repair coordinator to decide +/// whether the gateway process is up (reconnect only) or down (restart the distro). Never sends +/// credentials; it only opens and immediately closes a TCP connection to host:port. +/// +public static class GatewayReachabilityProbe +{ + public static async Task IsReachableAsync(string url, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(url) || !Uri.TryCreate(url, UriKind.Absolute, out var uri)) + return false; + + var host = uri.Host; + var port = uri.Port > 0 + ? uri.Port + : uri.Scheme.Equals("wss", StringComparison.OrdinalIgnoreCase) ? 443 : 80; + + try + { + using var client = new TcpClient(); + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeout.CancelAfter(TimeSpan.FromSeconds(2)); + await client.ConnectAsync(host, port, timeout.Token).ConfigureAwait(false); + return client.Connected; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch + { + return false; // refused / unreachable / timed out + } + } +} diff --git a/src/OpenClaw.Tray.WinUI/Services/ManagedLocalGatewayRepairCoordinator.cs b/src/OpenClaw.Tray.WinUI/Services/ManagedLocalGatewayRepairCoordinator.cs new file mode 100644 index 000000000..b33ac12c6 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Services/ManagedLocalGatewayRepairCoordinator.cs @@ -0,0 +1,510 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using OpenClaw.Connection; +using OpenClaw.Shared; + +namespace OpenClawTray.Services; + +/// Outcome of a managed-local gateway repair attempt. +public enum ManagedLocalGatewayRepairOutcome +{ + /// The active gateway is not an app-owned setup-managed local WSL gateway; nothing touched. + NotEligible, + + /// A repair is already running; this call coalesced. + AlreadyInProgress, + + /// The active gateway changed (user switch/disconnect) during the repair; aborted without + /// restarting or reconnecting a gateway we are no longer on. + AbortedGatewayChanged, + + /// The operator explicitly disconnected or stopped this gateway; automatic repair must + /// not override that desired state. + AbortedUserIntent, + + /// By the time a restart was due, the operator's failure was no longer a transport outage + /// (e.g. it became an auth/pairing/credential error). A distro restart cannot fix those, so the + /// repair aborts rather than pointlessly restarting and burning the restart budget. + AbortedNonTransportFailure, + + /// The per-gateway restart budget is exhausted; manual action needed. + AttemptsExhausted, + + /// The gateway process was already reachable; reconnected without restarting the distro. + ReconnectedWithoutRestart, + + /// Restarting the managed WSL distro failed. + RestartFailed, + + /// An unverified process owns the managed gateway port; no process was stopped. + PortConflictBlocked, + + /// A proven obsolete native OpenClaw gateway could not be removed safely. + PortConflictRepairFailed, + + /// Restart/reconnect ran but the operator connection was not verified before the timeout. + ReconnectPendingVerification, + + /// Restart + reconnect succeeded and the operator connection is verified live. + Repaired +} + +/// Structured result of a managed-local gateway repair. +public sealed record ManagedLocalGatewayRepairResult( + ManagedLocalGatewayRepairOutcome Outcome, + string? DistroName = null, + string? Detail = null) +{ + public bool Succeeded => + Outcome is ManagedLocalGatewayRepairOutcome.Repaired + or ManagedLocalGatewayRepairOutcome.ReconnectedWithoutRestart; +} + +/// Result of restarting a managed-local WSL gateway distro. +public sealed record ManagedLocalGatewayRestartResult(bool Success, string? Detail = null); + +/// Restarts an app-owned local WSL gateway distro. Abstracted for testability. +public interface IManagedLocalGatewayRestarter +{ + Task RestartAsync(string distroName, CancellationToken cancellationToken); +} + +/// +/// Bounded self-repair for app-owned setup-managed local WSL gateways. It probes before it +/// restarts: if the gateway process is already reachable, it reconnects without touching the +/// distro (the Mac "attach" path); only a genuinely-down gateway triggers a distro restart, a +/// keepalive re-arm, and a reconnect. Success is verified by a real operator connection to the +/// same gateway rather than trusting the restart command. +/// +/// +/// Safety invariants (this is the single owner of dangerous local process supervision, kept out +/// of the connection layer): +/// +/// Only acts on gateways — +/// never SSH, remote, or ambiguous-localhost gateways. +/// Probe-before-restart so a transient blip or a live-gateway/dead-socket case does not +/// trigger the heaviest remedy. +/// Reconnect is gateway-pinned and cancellable +/// () so a gateway switch during +/// repair cannot disrupt the newly selected gateway. +/// Single-flight; per-gateway restart budget (reset on verified success or gateway change, +/// and by the monitor when the gateway is healthy again). +/// Never reads or logs credential material. +/// +/// +public sealed class ManagedLocalGatewayRepairCoordinator +{ + private readonly GatewayRegistry _registry; + private readonly IManagedLocalGatewayRestarter _restarter; + private readonly Func> _probeReachableAsync; + private readonly Func> _reconnectIfCurrentAsync; + private readonly Func _isOperatorConnected; + private readonly Func _reArmKeepAliveAsync; + private readonly ConnectionDiagnostics _diagnostics; + private readonly IOpenClawLogger _logger; + private readonly Func _delay; + private readonly int _maxRestartsPerGateway; + private readonly TimeSpan _verifyTimeout; + private readonly TimeSpan _verifyPollInterval; + // Non-blocking acquire of the shared gateway-lifecycle lease; null => a manual op holds it, so the + // destructive restart must be deferred. Held only around the restart (mutual exclusion with a + // manual WSL start/stop/restart). + private readonly Func? _tryAcquireLifecycleLease; + // Re-checks, immediately before restarting, that the operator failure is STILL a transport outage + // (not an auth/pairing/credential error that surfaced during the verify wait — a restart can't fix + // those). Null => skip the re-check. + private readonly Func? _isRestartStillWarranted; + private readonly Func _isAutomaticRepairAllowed; + private readonly Func>? + _repairPortConflictAsync; + private readonly Func _isPortConflictCandidate; + + private readonly SemaphoreSlim _repairLock = new(1, 1); + private readonly object _attemptLock = new(); + // Restart budget keyed by gateway id so alternating between gateways cannot bypass the per-gateway + // bound (a single-slot counter would reset every time the target gateway changed). + private readonly Dictionary _restartCounts = new(StringComparer.Ordinal); + + public ManagedLocalGatewayRepairCoordinator( + GatewayRegistry registry, + IManagedLocalGatewayRestarter restarter, + Func> probeReachableAsync, + Func> reconnectIfCurrentAsync, + Func isOperatorConnected, + Func reArmKeepAliveAsync, + ConnectionDiagnostics diagnostics, + IOpenClawLogger logger, + int maxRestartsPerGateway = 3, + TimeSpan? verifyTimeout = null, + TimeSpan? verifyPollInterval = null, + Func? delay = null, + Func? tryAcquireLifecycleLease = null, + Func? isRestartStillWarranted = null, + Func? isAutomaticRepairAllowed = null, + Func>? + repairPortConflictAsync = null, + Func? isPortConflictCandidate = null) + { + _registry = registry ?? throw new ArgumentNullException(nameof(registry)); + _restarter = restarter ?? throw new ArgumentNullException(nameof(restarter)); + _probeReachableAsync = probeReachableAsync ?? throw new ArgumentNullException(nameof(probeReachableAsync)); + _reconnectIfCurrentAsync = reconnectIfCurrentAsync ?? throw new ArgumentNullException(nameof(reconnectIfCurrentAsync)); + _isOperatorConnected = isOperatorConnected ?? throw new ArgumentNullException(nameof(isOperatorConnected)); + _reArmKeepAliveAsync = reArmKeepAliveAsync ?? throw new ArgumentNullException(nameof(reArmKeepAliveAsync)); + _diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _maxRestartsPerGateway = maxRestartsPerGateway > 0 ? maxRestartsPerGateway : 1; + _verifyTimeout = verifyTimeout ?? TimeSpan.FromSeconds(15); + _verifyPollInterval = verifyPollInterval ?? TimeSpan.FromMilliseconds(250); + _delay = delay ?? Task.Delay; + _tryAcquireLifecycleLease = tryAcquireLifecycleLease; + _isRestartStillWarranted = isRestartStillWarranted; + _isAutomaticRepairAllowed = isAutomaticRepairAllowed ?? (static _ => true); + _repairPortConflictAsync = repairPortConflictAsync; + _isPortConflictCandidate = isPortConflictCandidate ?? (static () => false); + } + + /// + /// Repairs the active gateway if — and only if — it is an app-owned setup-managed local WSL + /// gateway. Safe to call when healthy or ineligible; returns a descriptive outcome without + /// side effects in those cases. + /// + public async Task TryRepairActiveGatewayAsync(CancellationToken cancellationToken = default) + { + if (!WslKeepAlivePolicy.IsSetupManagedLocalRecord(_registry.GetActive() ?? EmptyRecord)) + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.NotEligible); + + if (!await _repairLock.WaitAsync(0, cancellationToken).ConfigureAwait(false)) + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.AlreadyInProgress); + + IDisposable? lifecycleLease = null; + try + { + // Re-read the active record under the lock and bind everything below to THIS record so a + // gateway switch mid-repair cannot restart the wrong distro or report a false success. + var record = _registry.GetActive(); + if (record is null || !WslKeepAlivePolicy.IsSetupManagedLocalRecord(record)) + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.NotEligible); + + var distro = WslKeepAlivePolicy.ResolveDistroName(record, setupStateDistroName: null, environmentOverride: null); + var gatewayId = record.Id; + if (!_isAutomaticRepairAllowed(gatewayId)) + return new ManagedLocalGatewayRepairResult( + ManagedLocalGatewayRepairOutcome.AbortedUserIntent, + distro, + "Automatic repair is disabled because the gateway was explicitly disconnected or stopped."); + + var collisionRepaired = false; + if (_repairPortConflictAsync is not null && _isPortConflictCandidate()) + { + lifecycleLease = _tryAcquireLifecycleLease?.Invoke(); + if (_tryAcquireLifecycleLease is not null && lifecycleLease is null) + { + return new ManagedLocalGatewayRepairResult( + ManagedLocalGatewayRepairOutcome.AbortedGatewayChanged, + distro, + "Manual gateway operation in progress."); + } + + var currentBeforeRepair = _registry.GetActive(); + var currentDistroBeforeRepair = currentBeforeRepair is null + ? null + : WslKeepAlivePolicy.ResolveDistroName( + currentBeforeRepair, + setupStateDistroName: null, + environmentOverride: null); + if (currentBeforeRepair is null || + !string.Equals(currentBeforeRepair.Id, gatewayId, StringComparison.Ordinal) || + !WslKeepAlivePolicy.IsSetupManagedLocalRecord(currentBeforeRepair) || + !string.Equals(currentDistroBeforeRepair, distro, StringComparison.OrdinalIgnoreCase) || + !_isAutomaticRepairAllowed(gatewayId) || + !_isPortConflictCandidate()) + { + return new ManagedLocalGatewayRepairResult( + ManagedLocalGatewayRepairOutcome.AbortedUserIntent, + distro, + "Gateway changed or was explicitly stopped before port-conflict repair."); + } + + var portRepair = await _repairPortConflictAsync(record, cancellationToken).ConfigureAwait(false); + switch (portRepair.Outcome) + { + case ManagedLocalPortConflictRepairOutcome.Repaired: + collisionRepaired = true; + _diagnostics.Record("setup", "Removed proven obsolete native gateway port conflict", portRepair.Detail); + break; + case ManagedLocalPortConflictRepairOutcome.BlockedUnknownOwner: + _diagnostics.Record("setup", "Managed gateway port conflict has an unverified owner", portRepair.Detail); + return new ManagedLocalGatewayRepairResult( + ManagedLocalGatewayRepairOutcome.PortConflictBlocked, + distro, + portRepair.Detail); + case ManagedLocalPortConflictRepairOutcome.Failed: + _diagnostics.Record("setup", "Could not remove proven native gateway port conflict", portRepair.Detail); + return new ManagedLocalGatewayRepairResult( + ManagedLocalGatewayRepairOutcome.PortConflictRepairFailed, + distro, + portRepair.Detail); + } + } + + // Probe-before-restart: if the gateway process is already reachable, try a plain reconnect + // first (cheapest remedy, no restart budget consumed). If the gateway answers TCP but the + // connection still can't be verified — e.g. a wedged gateway that accepts sockets yet never + // completes the WebSocket handshake — fall through to a budgeted distro restart instead of + // looping on the TCP-positive reachable path forever. + if (!collisionRepaired && + await SafeProbeAsync(record.Url, cancellationToken).ConfigureAwait(false)) + { + if (!_isAutomaticRepairAllowed(gatewayId)) + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.AbortedUserIntent, distro); + + _diagnostics.Record("setup", "Local gateway reachable; reconnecting without restart", $"distro={distro}"); + var gatewayStillCurrent = true; + var verified = false; + try + { + // A false return means the gateway is no longer the active one (user switched or + // disconnected during the repair); true means we reconnected the pinned gateway. + gatewayStillCurrent = await _reconnectIfCurrentAsync(gatewayId, cancellationToken).ConfigureAwait(false); + if (gatewayStillCurrent) + verified = await VerifyAsync(gatewayId, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + // A reconnect that ERRORS (vs cleanly no-oping) means the pinned gateway is still + // current but the connect failed — leave gatewayStillCurrent=true so we escalate. + _logger.Warn($"[GatewayRepair] Reconnect (reachable path) threw: {ex.Message}"); + _diagnostics.Record("setup", "Local gateway reconnect (reachable) failed", ex.Message); + } + + if (!gatewayStillCurrent) + { + // Do NOT escalate to a distro restart of a gateway we are no longer on. + _diagnostics.Record("setup", "Active gateway changed during reconnect; aborting repair", $"distro={distro}"); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.AbortedGatewayChanged, distro, + "Active gateway changed during repair."); + } + + if (verified) + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.ReconnectedWithoutRestart, distro); + + // Reachable + still the pinned gateway + not verified: the gateway is answering TCP yet + // the operator connection is still down. Escalate to a budgeted distro restart rather + // than retrying the reachable path. + _diagnostics.Record("setup", "Local gateway reachable but not verified; escalating to restart", $"distro={distro}"); + } + + // The gateway process is down (or wedged) — this needs a distro restart, which is budgeted + // per gateway id. + // + // Re-validate the FULL record — not just the id — immediately before the destructive + // restart. During the up-to-15s verify the user may have (a) switched gateways, (b) + // repointed THIS record to a remote host or SSH tunnel (same id, no longer a managed WSL + // gateway), or (c) started a manual gateway lifecycle action. Any of these means we must not + // restart the cached distro (a manual restart + our --terminate could kill its fresh VM). + var currentActive = _registry.GetActive(); + var currentDistro = currentActive is null + ? null + : WslKeepAlivePolicy.ResolveDistroName(currentActive, setupStateDistroName: null, environmentOverride: null); + if (currentActive is null || + !string.Equals(currentActive.Id, gatewayId, StringComparison.Ordinal) || + !WslKeepAlivePolicy.IsSetupManagedLocalRecord(currentActive) || + !string.Equals(currentDistro, distro, StringComparison.OrdinalIgnoreCase)) + { + _diagnostics.Record("setup", "Gateway no longer eligible for restart (switched/edited); aborting", $"distro={distro}"); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.AbortedGatewayChanged, distro, + "Gateway changed during repair."); + } + if (!_isAutomaticRepairAllowed(gatewayId)) + { + _diagnostics.Record("setup", "Gateway was explicitly disconnected/stopped; skipping restart", $"distro={distro}"); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.AbortedUserIntent, distro); + } + + // The operator failure may have become a non-transport terminal error (auth/pairing/creds) + // during the up-to-15s verify wait — a distro restart cannot fix those, so abort instead of + // burning the restart budget on a pointless restart. + if (_isRestartStillWarranted is not null && !_isRestartStillWarranted()) + { + _diagnostics.Record("setup", "Operator failure is no longer a transport outage; skipping restart", $"distro={distro}"); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.AbortedNonTransportFailure, distro, + "Failure is not a transport outage; a restart cannot fix it."); + } + + // Acquire the shared gateway-lifecycle lease so this destructive restart is mutually + // exclusive with a manual WSL start/stop/restart. Non-blocking: if a manual op holds it, + // defer — running a concurrent restart (whose --terminate could kill the manual op's fresh + // VM) is exactly what this lease prevents. + lifecycleLease ??= _tryAcquireLifecycleLease?.Invoke(); + if (_tryAcquireLifecycleLease is not null && lifecycleLease is null) + { + _diagnostics.Record("setup", "Manual gateway operation in progress; deferring auto-repair restart", $"distro={distro}"); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.AbortedGatewayChanged, distro, + "Manual gateway operation in progress."); + } + + ManagedLocalGatewayRestartResult restart; + try + { + lock (_attemptLock) + { + _restartCounts.TryGetValue(gatewayId, out var count); + if (count >= _maxRestartsPerGateway) + { + _diagnostics.Record("setup", "Local gateway restart budget exhausted; manual action needed", $"distro={distro}; restarts={count}"); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.AttemptsExhausted, distro, + "Repair restart budget exhausted. Restart the gateway from Connection settings or re-run setup."); + } + + _restartCounts[gatewayId] = count + 1; + } + + _diagnostics.Record("setup", "Local gateway unreachable; restarting managed WSL distro", $"distro={distro}"); + + try + { + restart = await _restarter.RestartAsync(distro!, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.Warn($"[GatewayRepair] WSL restart threw: {ex.Message}"); + _diagnostics.Record("setup", "Local gateway restart failed", ex.Message); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.RestartFailed, distro, ex.Message); + } + + if (!restart.Success) + { + _diagnostics.Record("setup", "Local gateway restart failed", restart.Detail); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.RestartFailed, distro, restart.Detail); + } + + // Re-arm the WSL keepalive so an externally-terminated distro does not idle back out + // immediately after we bring it back. + try { await _reArmKeepAliveAsync(cancellationToken).ConfigureAwait(false); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } + catch (Exception ex) { _logger.Warn($"[GatewayRepair] Keepalive re-arm threw: {ex.Message}"); } + } + finally + { + // Release the lease before the (non-destructive) reconnect + verify so a manual op does + // not wait out the verify window. + lifecycleLease?.Dispose(); + lifecycleLease = null; + } + + _diagnostics.Record("setup", "Local gateway restarted; reconnecting", $"distro={distro}"); + cancellationToken.ThrowIfCancellationRequested(); + if (!_isAutomaticRepairAllowed(gatewayId)) + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.AbortedUserIntent, distro); + try + { + if (!await _reconnectIfCurrentAsync(gatewayId, cancellationToken).ConfigureAwait(false)) + { + // Active gateway changed after the restart — do not keep driving a gateway we are + // no longer on. The distro restart already happened (budget consumed); that's fine. + _diagnostics.Record("setup", "Active gateway changed after restart; aborting repair", $"distro={distro}"); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.AbortedGatewayChanged, distro, + "Active gateway changed during repair."); + } + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.Warn($"[GatewayRepair] Reconnect after restart threw: {ex.Message}"); + _diagnostics.Record("setup", "Local gateway reconnect after restart failed", ex.Message); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.ReconnectPendingVerification, distro, + "Gateway restarted; reconnect could not be completed."); + } + + if (await VerifyAsync(gatewayId, cancellationToken).ConfigureAwait(false)) + { + lock (_attemptLock) { _restartCounts.Remove(gatewayId); } + _diagnostics.Record("setup", "Local gateway repair verified (operator connected)", $"distro={distro}"); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.Repaired, distro); + } + + _diagnostics.Record("setup", "Local gateway restarted but operator connection not yet verified", $"distro={distro}"); + return new ManagedLocalGatewayRepairResult(ManagedLocalGatewayRepairOutcome.ReconnectPendingVerification, distro, + "Gateway restarted; still waiting for the connection to come back."); + } + finally + { + lifecycleLease?.Dispose(); + _repairLock.Release(); + } + } + + /// Resets the restart budget for a specific gateway. Called by the monitor when that + /// gateway is healthy again so a later outage starts fresh — without clearing other gateways' + /// budgets (which would let alternating gateways evade the per-gateway bound). + public void ResetAttemptBudget(string? gatewayId) + { + if (string.IsNullOrEmpty(gatewayId)) + return; + lock (_attemptLock) { _restartCounts.Remove(gatewayId); } + } + + /// Resets the restart budget for all gateways. + public void ResetAttemptBudget() + { + lock (_attemptLock) { _restartCounts.Clear(); } + } + + private async Task SafeProbeAsync(string? url, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(url)) + return false; + try + { + return await _probeReachableAsync(url, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.Warn($"[GatewayRepair] Reachability probe threw: {ex.Message}"); + return false; // treat probe failure as unreachable + } + } + + // Verified success requires BOTH that the operator is connected AND that the still-active gateway + // is the one we repaired — otherwise a gateway switch mid-repair could report a false success. + private async Task VerifyAsync(string gatewayId, CancellationToken cancellationToken) + { + bool Verified() => + string.Equals(_registry.ActiveGatewayId, gatewayId, StringComparison.Ordinal) && + _isOperatorConnected(); + + if (Verified()) + return true; + + var deadline = DateTimeOffset.UtcNow + _verifyTimeout; + while (DateTimeOffset.UtcNow < deadline) + { + cancellationToken.ThrowIfCancellationRequested(); + await _delay(_verifyPollInterval, cancellationToken).ConfigureAwait(false); + if (Verified()) + return true; + } + + return false; + } + + private static readonly GatewayRecord EmptyRecord = new() { Id = string.Empty, Url = string.Empty }; +} diff --git a/src/OpenClaw.Tray.WinUI/Services/NodeService.cs b/src/OpenClaw.Tray.WinUI/Services/NodeService.cs index ce05dc50f..464b36a95 100644 --- a/src/OpenClaw.Tray.WinUI/Services/NodeService.cs +++ b/src/OpenClaw.Tray.WinUI/Services/NodeService.cs @@ -109,6 +109,7 @@ public sealed class NodeService : IDisposable, IAsyncDisposable private readonly Func? _browserControlPortResolver; private readonly Func? _activeGatewayTunnelResolver; private readonly Func? _activeGatewayUrlResolver; + private readonly Func>? _browserControlAuthorization; private string? _token; // Authoritative capability list — populated by RegisterCapabilities and @@ -215,7 +216,8 @@ public NodeService( Func? sharedGatewayTokenResolver = null, Func? browserControlPortResolver = null, Func? activeGatewayTunnelResolver = null, - Func? activeGatewayUrlResolver = null) + Func? activeGatewayUrlResolver = null, + Func>? browserControlAuthorization = null) { _logger = logger; _dispatcherQueue = dispatcherQueue; @@ -225,6 +227,7 @@ public NodeService( _browserControlPortResolver = browserControlPortResolver; _activeGatewayTunnelResolver = activeGatewayTunnelResolver; _activeGatewayUrlResolver = activeGatewayUrlResolver; + _browserControlAuthorization = browserControlAuthorization; _rootProvider = rootProvider ?? (() => null); _chatProviderProvider = chatProviderProvider ?? (() => null); _inlineApprovalAvailable = inlineApprovalAvailable ?? (_ => false); @@ -459,7 +462,8 @@ private void RegisterCapabilities() controlPortOverride: _browserControlPortResolver?.Invoke(), useSshTunnel: tunnelState.Enabled, sshTunnelLocalPort: tunnelState.LocalPort, - allowGatewayPortFallback: tunnelState.AllowGatewayPortFallback); + allowGatewayPortFallback: tunnelState.AllowGatewayPortFallback, + authorizeEndpointAsync: _browserControlAuthorization); Register(_browserProxyCapability); } @@ -555,12 +559,14 @@ public void AttachClient(WindowsNodeClient client, string? bearerToken = null) // -= before += so a re-attach of the same client (e.g. AttachClient called // again after a DisconnectAsync that nulled _nodeClient) doesn't double-subscribe. client.StatusChanged -= OnNodeStatusChanged; + client.Disposed -= OnNodeClientDisposed; client.PairingStatusChanged -= OnPairingStatusChanged; client.HealthReceived -= OnNodeHealthReceived; client.GatewaySelfUpdated -= OnGatewaySelfUpdated; client.InvokeCompleted -= OnNodeInvokeCompleted; client.ToolTelemetryCompleted -= OnToolTelemetryCompleted; client.StatusChanged += OnNodeStatusChanged; + client.Disposed += OnNodeClientDisposed; client.PairingStatusChanged += OnPairingStatusChanged; client.HealthReceived += OnNodeHealthReceived; client.GatewaySelfUpdated += OnGatewaySelfUpdated; @@ -568,6 +574,15 @@ public void AttachClient(WindowsNodeClient client, string? bearerToken = null) client.ToolTelemetryCompleted += OnToolTelemetryCompleted; } + var gatewayUrl = client.GatewayUrl; + var configuredGatewayUrl = GetConfiguredGatewayUrl(); + var token = bearerToken; + _dispatcherQueue.TryEnqueue(() => + { + if (_canvasWindow != null && !_canvasWindow.IsClosed) + _canvasWindow.SetTrustedGatewayOrigin(gatewayUrl, token, configuredGatewayUrl); + }); + bool capabilitiesBuilt; lock (_capabilitiesLock) { @@ -595,6 +610,7 @@ public void AttachClient(WindowsNodeClient client, string? bearerToken = null) private void DetachClientHandlers(WindowsNodeClient client) { client.StatusChanged -= OnNodeStatusChanged; + client.Disposed -= OnNodeClientDisposed; client.PairingStatusChanged -= OnPairingStatusChanged; client.HealthReceived -= OnNodeHealthReceived; client.GatewaySelfUpdated -= OnGatewaySelfUpdated; @@ -1083,8 +1099,48 @@ private Dictionary BuildLocalPermissions() private void OnNodeStatusChanged(object? sender, ConnectionStatus status) { _logger.Info($"Node status changed: {status}"); + if (status == ConnectionStatus.Connected) + { + _dispatcherQueue.TryEnqueue(() => + { + if (_canvasWindow != null && !_canvasWindow.IsClosed) + _canvasWindow.SetTrustedGatewayOrigin(GatewayUrl, _token, GetConfiguredGatewayUrl()); + }); + } + else + { + _dispatcherQueue.TryEnqueue(() => + { + if (_canvasWindow != null && !_canvasWindow.IsClosed) + _canvasWindow.SetTrustedGatewayOrigin(null); + }); + } StatusChanged?.Invoke(this, status); } + + private void OnNodeClientDisposed(object? sender, EventArgs args) + { + var retired = false; + lock (_clientLock) + { + if (sender is WindowsNodeClient client && ReferenceEquals(_nodeClient, client)) + { + DetachClientHandlers(client); + _nodeClient = null; + _token = null; + retired = true; + } + } + + if (!retired) + return; + + _dispatcherQueue.TryEnqueue(() => + { + if (_canvasWindow != null && !_canvasWindow.IsClosed) + _canvasWindow.SetTrustedGatewayOrigin(null); + }); + } private void OnPairingStatusChanged(object? sender, PairingStatusEventArgs args) { diff --git a/src/OpenClaw.Tray.WinUI/Services/PortDiagnosticsService.cs b/src/OpenClaw.Tray.WinUI/Services/PortDiagnosticsService.cs index 3f92ab9d0..7781a9c3f 100644 --- a/src/OpenClaw.Tray.WinUI/Services/PortDiagnosticsService.cs +++ b/src/OpenClaw.Tray.WinUI/Services/PortDiagnosticsService.cs @@ -1,10 +1,10 @@ +using OpenClaw.Connection; using OpenClaw.Shared; using System; using System.Collections.Generic; using System.Diagnostics; using System.Linq; using System.Net.NetworkInformation; -using System.Runtime.InteropServices; namespace OpenClawTray.Services; @@ -187,78 +187,11 @@ private static string FormatProcessName(string? processName) => private static IEnumerable GetWindowsTcpListenerOwners() { - if (!OperatingSystem.IsWindows()) - yield break; - - var bufferLength = 0; - var result = GetExtendedTcpTable( - IntPtr.Zero, - ref bufferLength, - sort: true, - ipVersion: AfInet, - tableClass: TcpTableOwnerPidListener, - reserved: 0); - if (result != ErrorInsufficientBuffer || bufferLength <= 0) - yield break; - - var tablePtr = Marshal.AllocHGlobal(bufferLength); - try - { - result = GetExtendedTcpTable( - tablePtr, - ref bufferLength, - sort: true, - ipVersion: AfInet, - tableClass: TcpTableOwnerPidListener, - reserved: 0); - if (result != ErrorSuccess) - yield break; - - var rowCount = Marshal.ReadInt32(tablePtr); - var rowPtr = IntPtr.Add(tablePtr, sizeof(int)); - var rowSize = Marshal.SizeOf(); - for (var i = 0; i < rowCount; i++) - { - var row = Marshal.PtrToStructure(rowPtr); - var port = (row.LocalPort[0] << 8) + row.LocalPort[1]; - if (port is >= 1 and <= 65535) - yield return new TcpListenerProcessOwner(port, unchecked((int)row.OwningProcessId)); - rowPtr = IntPtr.Add(rowPtr, rowSize); - } - } - finally - { - Marshal.FreeHGlobal(tablePtr); - } + foreach (var listener in WindowsTcpListenerSnapshot.Capture().Listeners) + yield return new TcpListenerProcessOwner(listener.Port, listener.ProcessId); } private readonly record struct TcpListenerOwner(int ProcessId, string? ProcessName); private readonly record struct TcpListenerProcessOwner(int Port, int ProcessId); - private const int AfInet = 2; - private const int TcpTableOwnerPidListener = 3; - private const uint ErrorSuccess = 0; - private const uint ErrorInsufficientBuffer = 122; - - [DllImport("iphlpapi.dll", SetLastError = true)] - private static extern uint GetExtendedTcpTable( - IntPtr tcpTable, - ref int tcpTableLength, - bool sort, - int ipVersion, - int tableClass, - uint reserved); - - [StructLayout(LayoutKind.Sequential)] - private struct MibTcpRowOwnerPid - { - public uint State; - public uint LocalAddress; - [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] - public byte[] LocalPort; - public uint RemoteAddress; - [MarshalAs(UnmanagedType.ByValArray, SizeConst = 4)] - public byte[] RemotePort; - public uint OwningProcessId; - } } diff --git a/src/OpenClaw.Tray.WinUI/Services/SettingsManager.cs b/src/OpenClaw.Tray.WinUI/Services/SettingsManager.cs index f211c7b55..60359e633 100644 --- a/src/OpenClaw.Tray.WinUI/Services/SettingsManager.cs +++ b/src/OpenClaw.Tray.WinUI/Services/SettingsManager.cs @@ -139,6 +139,8 @@ public List UserRules public string TtsPiperVoiceId { get => string.IsNullOrWhiteSpace(_data.TtsPiperVoiceId) ? "en_US-amy-low" : _data.TtsPiperVoiceId; set => _data = _data with { TtsPiperVoiceId = value }; } // Local MCP HTTP server (independent of EnableNodeMode) public bool EnableMcpServer { get => _data.EnableMcpServer; set => _data = _data with { EnableMcpServer = value }; } + // Automatic self-repair of app-owned setup-managed local WSL gateways (kill switch). + public bool EnableManagedLocalGatewayAutoRepair { get => _data.EnableManagedLocalGatewayAutoRepair; set => _data = _data with { EnableManagedLocalGatewayAutoRepair = value }; } /// /// Hostnames the A2UI image renderer is allowed to fetch over HTTPS. /// Empty by default — agents can still ship inline data: images. The diff --git a/src/OpenClaw.Tray.WinUI/Services/WslGatewayKeepAliveService.cs b/src/OpenClaw.Tray.WinUI/Services/WslGatewayKeepAliveService.cs index 07a3a1e10..6d613a54c 100644 --- a/src/OpenClaw.Tray.WinUI/Services/WslGatewayKeepAliveService.cs +++ b/src/OpenClaw.Tray.WinUI/Services/WslGatewayKeepAliveService.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Text.Json; using System.Threading.Tasks; namespace OpenClawTray.Services; @@ -52,7 +53,18 @@ public async Task TryEnsureAsync() return; } - // Spawn a detached wsl sleep process to keep the VM alive + // Spawn a detached wsl sleep process to keep the VM alive. Idempotent: TryEnsureAsync runs + // at startup AND on every managed-local auto-repair (re-arm after a distro restart), so a + // gateway-only outage where the VM stays up would otherwise leak a new detached wsl.exe sleep + // process on each repair. Skip the spawn when a keepalive for this distro already exists. + var localDataDir = SetupExistingGatewayClassifier.ResolveLocalDataPath(); + var markerPath = Path.Combine(localDataDir, "wsl-keepalive", $"{distroName}.json"); + if (IsKeepAliveRunningForDistro(distroName, markerPath)) + { + Logger.Info($"[WslKeepAlive] Keepalive already running for {distroName}; skipping spawn."); + return; + } + var psi = new System.Diagnostics.ProcessStartInfo { FileName = ResolveWslExePath(), @@ -69,6 +81,11 @@ public async Task TryEnsureAsync() if (proc is not null) { Logger.Info($"[WslKeepAlive] Started keepalive for {distroName} (PID {proc.Id})."); + WriteKeepAliveMarker( + markerPath, + distroName, + proc.Id, + proc.StartTime.ToUniversalTime()); } } catch (Exception ex) @@ -130,6 +147,122 @@ private static IReadOnlyList ReadKeepAliveMarkerDistroNames(string marke : null; } + private static bool IsKeepAliveRunningForDistro(string distroName, string markerPath) + { + if (TryGetMarkedKeepAlive(markerPath, distroName)) + return true; + + var procs = System.Diagnostics.Process.GetProcessesByName("wsl") + .Concat(System.Diagnostics.Process.GetProcessesByName("wsl.exe")) + .ToArray(); + + try + { + foreach (var proc in procs) + { + try + { + var commandLine = GetProcessCommandLine(proc.Id); + + // Unreadable unrelated WSL processes are not proof that our keepalive exists. + // Owned keepalives have a PID marker, checked above. + if (commandLine is null) + continue; + if (WslKeepAlivePolicy.IsKeepaliveCommandLine(commandLine, distroName)) + return true; // positively found + } + // slopwatch-ignore: SW003 Inspection is best-effort; a process may exit mid-enumeration and that cannot improve caller state. + catch + { + // Process exited or is protected; continue looking for a positive match. + } + } + + // Every live wsl process was readable and none was a keepalive for this distro — safe to + // spawn. A genuinely idled-out VM has no wsl processes at all and also lands here. + return false; + } + finally + { + foreach (var proc in procs) + { + // slopwatch-ignore: SW003 Best-effort disposal of enumerated process handles. + try { proc.Dispose(); } catch { } + } + } + } + + private static bool TryGetMarkedKeepAlive(string markerPath, string distroName) + { + if (!File.Exists(markerPath)) + return false; + + try + { + using var doc = JsonDocument.Parse(File.ReadAllText(markerPath)); + if (!doc.RootElement.TryGetProperty("DistroName", out var distroElement) || + !string.Equals(distroElement.GetString(), distroName, StringComparison.OrdinalIgnoreCase) || + !doc.RootElement.TryGetProperty("Pid", out var pidElement) || + !pidElement.TryGetInt32(out var pid) || + !doc.RootElement.TryGetProperty("StartTimeUtc", out var startElement) || + !startElement.TryGetDateTime(out var markerStartTimeUtc)) + { + return false; + } + + using var process = System.Diagnostics.Process.GetProcessById(pid); + if (process.HasExited || + !WslKeepAlivePolicy.IsMarkedKeepaliveProcessIdentity( + process.ProcessName, + process.StartTime.ToUniversalTime(), + markerStartTimeUtc)) + return false; + + // The marker was written only after this service/setup spawned the process. A transient + // CIM failure therefore remains safe to treat as running for this exact owned PID. + var commandLine = GetProcessCommandLine(pid); + return commandLine is null || + WslKeepAlivePolicy.IsKeepaliveCommandLine(commandLine, distroName); + } + catch + { + return false; + } + } + + private static void WriteKeepAliveMarker( + string markerPath, + string distroName, + int pid, + DateTime processStartTimeUtc) + { + try + { + Directory.CreateDirectory(Path.GetDirectoryName(markerPath)!); + var json = JsonSerializer.Serialize(new + { + DistroName = distroName, + Pid = pid, + StartTimeUtc = processStartTimeUtc, + ProcessName = "wsl" + }); + var tempPath = markerPath + ".tmp-" + Guid.NewGuid().ToString("N"); + try + { + File.WriteAllText(tempPath, json); + File.Move(tempPath, markerPath, overwrite: true); + } + finally + { + try { File.Delete(tempPath); } catch { } + } + } + catch (Exception ex) + { + Logger.Warn($"[WslKeepAlive] Could not persist keepalive marker: {ex.Message}"); + } + } + private static void StopKeepAliveProcessesForDistro(string distroName) { var procs = System.Diagnostics.Process.GetProcessesByName("wsl") @@ -195,9 +328,19 @@ private static void DeleteKeepAliveMarker(string markerDir, string distroName) }; using var p = System.Diagnostics.Process.Start(psi); if (p == null) return null; - var output = p.StandardOutput.ReadToEnd(); - p.WaitForExit(5000); - return output.Trim(); + + // Drain stdout asynchronously so a large command line cannot deadlock the fixed-size pipe, + // and bound the whole inspection: WaitForExit(5000) returns before ReadToEnd could block + // forever on a hung CIM/PowerShell. On timeout, kill and report indeterminate (null). + var readTask = p.StandardOutput.ReadToEndAsync(); + if (!p.WaitForExit(5000)) + { + // slopwatch-ignore: SW003 Best-effort kill of a stuck inspection process; failure cannot improve caller state. + try { p.Kill(entireProcessTree: true); } catch { } + return null; + } + + return readTask.GetAwaiter().GetResult()?.Trim(); } catch { return null; } } diff --git a/src/OpenClaw.Tray.WinUI/Services/WslKeepAlivePolicy.cs b/src/OpenClaw.Tray.WinUI/Services/WslKeepAlivePolicy.cs index 7ef3fa3a9..edffd2f9d 100644 --- a/src/OpenClaw.Tray.WinUI/Services/WslKeepAlivePolicy.cs +++ b/src/OpenClaw.Tray.WinUI/Services/WslKeepAlivePolicy.cs @@ -28,8 +28,9 @@ public static bool ShouldStart(GatewayRecord? activeRecord, string? legacyGatewa string? setupStateDistroName, string? environmentOverride) { - if (!string.IsNullOrWhiteSpace(activeRecord?.SetupManagedDistroName)) - return activeRecord.SetupManagedDistroName; + if (activeRecord is not null && + GatewayRecordEditing.ResolveManagedDistroName(activeRecord) is { } managedDistroName) + return managedDistroName; if (!string.IsNullOrWhiteSpace(setupStateDistroName)) return setupStateDistroName; @@ -75,6 +76,17 @@ public static bool IsKeepaliveCommandLine(string? commandLine, string distroName return WslCommandLineMatcher.IsKeepaliveForDistro(commandLine, distroName); } + public static bool IsMarkedKeepaliveProcessIdentity( + string? processName, + DateTime processStartTimeUtc, + DateTime markerStartTimeUtc) + { + return string.Equals(processName, "wsl", StringComparison.OrdinalIgnoreCase) && + Math.Abs( + (processStartTimeUtc - + DateTime.SpecifyKind(markerStartTimeUtc, DateTimeKind.Utc)).TotalSeconds) <= 5; + } + public static bool TryGetMarkerDistroName(string markerJson, out string distroName) { distroName = string.Empty; @@ -99,21 +111,14 @@ public static bool IsSetupManagedLocalRecord(GatewayRecord record) if (record.SshTunnel is not null) return false; - if (!string.IsNullOrWhiteSpace(record.SetupManagedDistroName)) + if (GatewayRecordEditing.ResolveManagedDistroName(record) is not null) return record.IsLocal || LocalGatewayUrlClassifier.IsLocalGatewayUrl(record.Url); return IsLegacyDefaultSetupManagedLocalRecord(record); } private static string? GetSetupManagedDistroName(GatewayRecord record) - { - if (!string.IsNullOrWhiteSpace(record.SetupManagedDistroName)) - return record.SetupManagedDistroName; - - return IsLegacyDefaultSetupManagedLocalRecord(record) - ? DefaultSetupManagedDistroName - : null; - } + => GatewayRecordEditing.ResolveManagedDistroName(record); private static bool IsLegacyDefaultSetupManagedLocalRecord(GatewayRecord record) => record.IsLocal diff --git a/src/OpenClaw.Tray.WinUI/Windows/CanvasWindow.xaml.cs b/src/OpenClaw.Tray.WinUI/Windows/CanvasWindow.xaml.cs index 1dc2caf60..f9165d79a 100644 --- a/src/OpenClaw.Tray.WinUI/Windows/CanvasWindow.xaml.cs +++ b/src/OpenClaw.Tray.WinUI/Windows/CanvasWindow.xaml.cs @@ -154,7 +154,16 @@ private static bool IsSafeDataUrl(string url) /// public void SetTrustedGatewayOrigin(string? gatewayUrl, string? token = null, string? configuredGatewayUrl = null) { - if (string.IsNullOrEmpty(gatewayUrl)) return; + if (string.IsNullOrEmpty(gatewayUrl)) + { + _gatewayToken = null; + _trustedGatewayOrigin = null; + _configuredGatewayOrigin = null; + _gatewayOriginForRewrite = null; + if (CanvasWebView.CoreWebView2 != null) + RemoveGatewayAuthHeaderInjection(CanvasWebView.CoreWebView2); + return; + } _gatewayToken = token; try { diff --git a/src/OpenClaw.Tray.WinUI/Windows/ConnectionStatusWindow.xaml.cs b/src/OpenClaw.Tray.WinUI/Windows/ConnectionStatusWindow.xaml.cs index 07a99e56a..e8624e5b2 100644 --- a/src/OpenClaw.Tray.WinUI/Windows/ConnectionStatusWindow.xaml.cs +++ b/src/OpenClaw.Tray.WinUI/Windows/ConnectionStatusWindow.xaml.cs @@ -335,7 +335,7 @@ private void OnDisconnectClick(object sender, RoutedEventArgs e) => private async Task OnDisconnectClickAsync() { if (_manager == null) return; - await _manager.DisconnectAsync(); + await _manager.DisconnectByUserAsync(); SetupCodeResult.Text = LocalizationHelper.GetString("ConnectionStatus_Disconnected"); } diff --git a/tests/OpenClaw.Connection.Tests/ConnectionStateMachineTests.cs b/tests/OpenClaw.Connection.Tests/ConnectionStateMachineTests.cs index dcf71a49f..8f174e9f4 100644 --- a/tests/OpenClaw.Connection.Tests/ConnectionStateMachineTests.cs +++ b/tests/OpenClaw.Connection.Tests/ConnectionStateMachineTests.cs @@ -132,6 +132,18 @@ public void Connecting_AuthenticationFailed_TransitionsToError() Assert.Equal("bad token", _sm.Current.OperatorError); } + [Fact] + public void TypedOperatorFailureKind_IsPreservedInSnapshot_AndClearedOnReconnect() + { + _sm.TryTransition(ConnectionTrigger.ConnectRequested); + _sm.SetOperatorErrorKind(OpenClaw.Shared.GatewayErrorKind.Tls); + Assert.True(_sm.TryTransition(ConnectionTrigger.WebSocketError, "Transport error")); + Assert.Equal(OpenClaw.Shared.GatewayErrorKind.Tls, _sm.Current.OperatorErrorKind); + + Assert.True(_sm.TryTransition(ConnectionTrigger.ReconnectScheduled)); + Assert.Null(_sm.Current.OperatorErrorKind); + } + [Fact] public void Connecting_RateLimited_TransitionsToError() { diff --git a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs index 96326c7a1..114072b06 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs @@ -408,6 +408,33 @@ public async Task ApplySetupCodeAsync_SaveFailure_PreservesActiveGatewayAndLiveC Assert.Null(registry.FindByUrl("wss://test2")); } + [Fact] + public async Task ApplySetupCodeAsync_ClearsPriorDisconnectedIntent() + { + SetupGateway("gw-1", "wss://test1"); + _manager.SetGatewayConnectionIntent("gw-1", shouldBeConnected: false); + + var result = await _manager.ApplySetupCodeAsync( + BuildSetupCode("wss://test1", "bootstrap-token")); + + Assert.Equal(SetupCodeOutcome.Success, result.Outcome); + Assert.True(_manager.IsAutomaticReconnectAllowed("gw-1")); + } + + [Fact] + public async Task ConnectWithSharedTokenAsync_ClearsPriorDisconnectedIntent() + { + SetupGateway("gw-1", "wss://test1"); + _manager.SetGatewayConnectionIntent("gw-1", shouldBeConnected: false); + + var result = await _manager.ConnectWithSharedTokenAsync( + "wss://test1", + "shared-token"); + + Assert.Equal(SetupCodeOutcome.Success, result.Outcome); + Assert.True(_manager.IsAutomaticReconnectAllowed("gw-1")); + } + [Fact] public async Task StaleOldGatewayHandshakeAfterSwitch_DoesNotMutateCurrentSnapshot() { @@ -1294,6 +1321,568 @@ public async Task AuthenticationFailed_DeviceTokenMismatchAfterSuccessfulRecover Assert.True(factory.CreatedCredentials[3].IsBootstrapToken); } + [Fact] + public async Task AuthenticationFailed_DeviceTokenMismatch_SharedTokenFallback_RecoversWithSharedToken() + { + // Post-setup dead end: bootstrap token cleared once pairing is durable, but the shared + // gateway token remains. A later stale device token must still self-recover via shared. + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + SharedGatewayToken = "shared-token" + }); + _registry.SetActive("gw-local"); + + var identityDir = _registry.GetIdentityDirectory("gw-local"); + var identity = new DeviceIdentity(identityDir, NullLogger.Instance); + identity.Initialize(); + identity.StoreDeviceToken("stale-device-token"); + + var resolver = new CredentialResolver(new DeviceIdentityFileReader()); + var factory = new MockClientFactory(); + using var manager = new GatewayConnectionManager( + resolver, factory, _registry, NullLogger.Instance, + reconnectDelay: _ => Task.CompletedTask, + endpointProvenanceProbe: (_, _) => Task.FromResult( + new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ExpectedManagedGateway, + 18789))); + + await manager.ConnectAsync("gw-local"); + Assert.Equal(CredentialResolver.SourceDeviceToken, factory.CreatedCredentials[0].Source); + + factory.CreatedClients[0].SimulateAuthFailed("AUTH_DEVICE_TOKEN_MISMATCH"); + await WaitUntilAsync(() => factory.CreatedCredentials.Count >= 2); + + Assert.Null(DeviceIdentity.TryReadStoredDeviceToken(identityDir)); + Assert.Equal(CredentialResolver.SourceSharedGatewayToken, factory.CreatedCredentials[1].Source); + } + + [Fact] + public async Task AuthenticationFailed_DeviceTokenMismatch_UntrustedPlainWsRemote_DoesNotRecover() + { + // SECURITY: over a plain ws:// remote (not loopback, not wss, not an owned tunnel) the + // manager must NOT clear the device token and downgrade to the stronger shared/bootstrap + // credential — a hostile cleartext endpoint could otherwise induce credential disclosure. + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-remote", + Url = "ws://remote.example:18789", + BootstrapToken = "bootstrap-token", + SharedGatewayToken = "shared-token" + }); + _registry.SetActive("gw-remote"); + + var identityDir = _registry.GetIdentityDirectory("gw-remote"); + var identity = new DeviceIdentity(identityDir, NullLogger.Instance); + identity.Initialize(); + identity.StoreDeviceToken("stale-device-token"); + + var resolver = new CredentialResolver(new DeviceIdentityFileReader()); + var factory = new MockClientFactory(); + using var manager = new GatewayConnectionManager( + resolver, factory, _registry, NullLogger.Instance, + reconnectDelay: _ => Task.CompletedTask); + + await manager.ConnectAsync("gw-remote"); + factory.CreatedClients[0].SimulateAuthFailed("AUTH_DEVICE_TOKEN_MISMATCH"); + await Task.Delay(150); + + Assert.Equal("stale-device-token", DeviceIdentity.TryReadStoredDeviceToken(identityDir)); + Assert.Single(factory.CreatedCredentials); + } + + [Fact] + public async Task AuthenticationFailed_DeviceTokenMismatch_UnknownManagedLoopbackOwner_DoesNotRecover() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + SharedGatewayToken = "shared-token" + }); + _registry.SetActive("gw-local"); + + var identityDir = _registry.GetIdentityDirectory("gw-local"); + var identity = new DeviceIdentity(identityDir, NullLogger.Instance); + identity.Initialize(); + identity.StoreDeviceToken("stale-device-token"); + + var resolver = new CredentialResolver(new DeviceIdentityFileReader()); + var factory = new MockClientFactory(); + using var manager = new GatewayConnectionManager( + resolver, factory, _registry, NullLogger.Instance, + reconnectDelay: _ => Task.CompletedTask, + endpointProvenanceProbe: (_, _) => Task.FromResult(new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + 18789, + ProcessId: 42, + ProcessName: "unknown"))); + + await manager.ConnectAsync("gw-local"); + factory.CreatedClients[0].SimulateAuthFailed("AUTH_DEVICE_TOKEN_MISMATCH"); + await Task.Delay(150); + + Assert.Equal("stale-device-token", DeviceIdentity.TryReadStoredDeviceToken(identityDir)); + Assert.Single(factory.CreatedCredentials); + } + + [Fact] + public async Task ManagedLoopback_UnknownOwner_StrongCredentialIsBlockedBeforeClientCreation() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + SharedGatewayToken = "shared-token" + }); + _registry.SetActive("gw-local"); + var resolver = new CredentialResolver(new DeviceIdentityFileReader()); + var factory = new MockClientFactory(); + using var manager = new GatewayConnectionManager( + resolver, + factory, + _registry, + NullLogger.Instance, + endpointProvenanceProbe: (_, _) => Task.FromResult(new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + 18789, + ProcessId: 42, + ProcessName: "unknown"))); + + await manager.ConnectAsync("gw-local"); + + Assert.Empty(factory.CreatedClients); + Assert.Equal(RoleConnectionState.Error, manager.CurrentSnapshot.OperatorState); + Assert.Equal(GatewayErrorKind.LocalPortConflict, manager.CurrentSnapshot.OperatorErrorKind); + } + + [Fact] + public async Task DisposeDuringSlowProvenanceProbe_NeverCreatesClientAfterDispose() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + SharedGatewayToken = "shared-token" + }); + _registry.SetActive("gw-local"); + var started = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var resolver = new CredentialResolver(new DeviceIdentityFileReader()); + var factory = new MockClientFactory(); + var manager = new GatewayConnectionManager( + resolver, + factory, + _registry, + NullLogger.Instance, + endpointProvenanceProbe: async (_, _) => + { + started.TrySetResult(); + await release.Task; + return new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ExpectedManagedGateway, + 18789); + }); + + var connect = manager.ConnectAsync("gw-local"); + await started.Task; + await manager.DisposeAsync(); + release.TrySetResult(); + try { await connect; } catch (ObjectDisposedException) { } + + Assert.Empty(factory.CreatedClients); + } + + [Fact] + public async Task LegacyManagedLoopback_UnknownOwner_StrongCredentialIsBlockedBeforeClientCreation() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-legacy", + Url = "ws://localhost:18789", + FriendlyName = "Local (OpenClawGateway)", + IsLocal = true, + SharedGatewayToken = "shared-token" + }); + _registry.SetActive("gw-legacy"); + var resolver = new CredentialResolver(new DeviceIdentityFileReader()); + var factory = new MockClientFactory(); + using var manager = new GatewayConnectionManager( + resolver, + factory, + _registry, + NullLogger.Instance, + endpointProvenanceProbe: (_, _) => Task.FromResult( + new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + 18789))); + + await manager.ConnectAsync("gw-legacy"); + + Assert.Empty(factory.CreatedClients); + Assert.Equal(GatewayErrorKind.LocalPortConflict, manager.CurrentSnapshot.OperatorErrorKind); + } + + [Fact] + public async Task NormalNodeStartup_UnknownOwner_SharedFallbackNeverReachesNodeConnector() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + SharedGatewayToken = "shared-token" + }); + _registry.SetActive("gw-local"); + var identityDir = _registry.GetIdentityDirectory("gw-local"); + var identity = new DeviceIdentity(identityDir, NullLogger.Instance); + identity.Initialize(); + identity.StoreDeviceTokenForRole("operator", "operator-device-token"); + var resolver = new CredentialResolver(new DeviceIdentityFileReader()); + var factory = new MockClientFactory(); + var node = new ScriptedNodeConnector(); + using var manager = new GatewayConnectionManager( + resolver, + factory, + _registry, + NullLogger.Instance, + nodeConnector: node, + isNodeEnabled: () => true, + endpointProvenanceProbe: (_, _) => Task.FromResult( + new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + 18789))); + + await manager.ConnectAsync("gw-local"); + factory.CreatedClients[0].SimulateHandshake(); + await WaitUntilAsync(() => manager.CurrentSnapshot.NodeState == RoleConnectionState.Error); + + Assert.Equal(0, node.ConnectCount); + } + + [Fact] + public async Task ManagedTailscale_DeviceMismatch_StillRecoversWithBootstrap() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-ts", + Url = "wss://host.tailnet.ts.net", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + BootstrapToken = "bootstrap-token" + }); + _registry.SetActive("gw-ts"); + var identityDir = _registry.GetIdentityDirectory("gw-ts"); + var identity = new DeviceIdentity(identityDir, NullLogger.Instance); + identity.Initialize(); + identity.StoreDeviceToken("stale-device-token"); + var resolver = new CredentialResolver(new DeviceIdentityFileReader()); + var factory = new MockClientFactory(); + using var manager = new GatewayConnectionManager( + resolver, + factory, + _registry, + NullLogger.Instance, + reconnectDelay: _ => Task.CompletedTask, + endpointProvenanceProbe: (_, _) => Task.FromResult(new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.NotApplicable, + 0))); + + await manager.ConnectAsync("gw-ts"); + factory.CreatedClients[0].SimulateAuthFailed("AUTH_DEVICE_TOKEN_MISMATCH"); + await WaitUntilAsync(() => factory.CreatedCredentials.Count >= 2); + + Assert.Equal(CredentialResolver.SourceBootstrapToken, factory.CreatedCredentials[1].Source); + } + + [Fact] + public async Task TypedOperatorTlsFailure_IsPreservedInSnapshot() + { + SetupGateway("gw-1", "wss://gateway.example"); + _resolver.OperatorCredential = new GatewayCredential("tok", false, "test"); + await _manager.ConnectAsync("gw-1"); + + _factory.CreatedClients[0].SimulateConnectionFailure(GatewayErrorKind.Tls); + _factory.CreatedClients[0].SimulateStatusChanged(ConnectionStatus.Error); + + await WaitUntilAsync(() => _manager.CurrentSnapshot.OperatorState == RoleConnectionState.Error); + Assert.Equal(GatewayErrorKind.Tls, _manager.CurrentSnapshot.OperatorErrorKind); + } + + [Fact] + public async Task SharedTokenMismatch_FromUnknownManagedLoopbackOwner_BecomesPortConflict() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + SharedGatewayToken = "shared-token" + }); + _registry.SetActive("gw-local"); + var resolver = new MockCredentialResolver + { + OperatorCredential = new GatewayCredential("shared-token", false, "test") + }; + var factory = new MockClientFactory(); + using var manager = new GatewayConnectionManager( + resolver, + factory, + _registry, + NullLogger.Instance, + endpointProvenanceProbe: (_, _) => Task.FromResult(new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + 18789, + ProcessId: 42, + ProcessName: "unknown"))); + + await manager.ConnectAsync("gw-local"); + factory.CreatedClients[0].SimulateConnectionFailure(GatewayErrorKind.Auth); + factory.CreatedClients[0].SimulateAuthFailed( + "unauthorized: gateway token mismatch (set gateway.remote.token to match gateway.auth.token)"); + factory.CreatedClients[0].SimulateStatusChanged(ConnectionStatus.Error); + + await WaitUntilAsync(() => manager.CurrentSnapshot.OperatorState == RoleConnectionState.Error); + Assert.Equal(GatewayErrorKind.LocalPortConflict, manager.CurrentSnapshot.OperatorErrorKind); + } + + [Fact] + public async Task CodeOnlyAuthFailure_FromProvenConflictingOwner_BecomesPortConflict() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway" + }); + _registry.SetActive("gw-local"); + var resolver = new MockCredentialResolver + { + OperatorCredential = new GatewayCredential("device-token", false, CredentialResolver.SourceDeviceToken) + }; + var factory = new MockClientFactory(); + using var manager = new GatewayConnectionManager( + resolver, + factory, + _registry, + NullLogger.Instance, + endpointProvenanceProbe: (_, _) => Task.FromResult(new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ConflictingOpenClawGateway, + 18789, + ProcessId: 42, + ProcessName: "node"))); + + await manager.ConnectAsync("gw-local"); + factory.CreatedClients[0].SimulateConnectionFailure(GatewayErrorKind.Auth); + factory.CreatedClients[0].SimulateAuthFailed("unauthorized"); + + await WaitUntilAsync(() => manager.CurrentSnapshot.OperatorState == RoleConnectionState.Error); + Assert.Equal(GatewayErrorKind.LocalPortConflict, manager.CurrentSnapshot.OperatorErrorKind); + } + + [Fact] + public async Task NodeConnectionFailure_DeviceTokenMismatch_ClearsOnlyNodeTokenAndReconnects() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + SharedGatewayToken = "shared-token" + }); + _registry.SetActive("gw-local"); + + var identityDir = _registry.GetIdentityDirectory("gw-local"); + var identity = new DeviceIdentity(identityDir, NullLogger.Instance); + identity.Initialize(); + identity.StoreDeviceTokenForRole("operator", "op-device-token", ["operator.read"]); + identity.StoreDeviceTokenForRole("node", "stale-node-token"); + + var resolver = new CredentialResolver(new DeviceIdentityFileReader()); + var factory = new MockClientFactory(); + var node = new ScriptedNodeConnector + { + ConnectAction = (n, _) => + { + n.SimulateStatus(ConnectionStatus.Connected); + n.SimulatePairing(PairingStatus.Paired); + } + }; + using var manager = new GatewayConnectionManager( + resolver, factory, _registry, NullLogger.Instance, + nodeConnector: node, + isNodeEnabled: () => true, + reconnectDelay: _ => Task.CompletedTask, + endpointProvenanceProbe: (_, _) => Task.FromResult( + new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ExpectedManagedGateway, + 18789))); + + await manager.ConnectAsync("gw-local"); + await InvokeHandshakeSucceededAsync(manager); + await WaitUntilAsync(() => node.ConnectCount >= 1); + var before = node.ConnectCount; + + node.SimulateConnectionFailure(GatewayErrorKind.DeviceTokenMismatch); + await WaitUntilAsync(() => node.ConnectCount > before); + + // Only the node device token is cleared; the operator device token is preserved. + Assert.Null(DeviceIdentity.TryReadStoredDeviceTokenForRole(identityDir, "node")); + Assert.Equal("op-device-token", DeviceIdentity.TryReadStoredDeviceTokenForRole(identityDir, "operator")); + + // With the stale node device token gone, the recovery reconnect must fall back to the shared + // gateway token — not silently keep failing on a credential that no longer exists. + Assert.Equal(CredentialResolver.SourceSharedGatewayToken, node.LastCredential?.Source); + } + + [Fact] + public async Task NodeConnectionFailure_NonDeviceKind_DoesNotClearNodeToken() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SharedGatewayToken = "shared-token" + }); + _registry.SetActive("gw-local"); + + var identityDir = _registry.GetIdentityDirectory("gw-local"); + var identity = new DeviceIdentity(identityDir, NullLogger.Instance); + identity.Initialize(); + identity.StoreDeviceTokenForRole("operator", "op-device-token", ["operator.read"]); + identity.StoreDeviceTokenForRole("node", "stale-node-token"); + + var resolver = new CredentialResolver(new DeviceIdentityFileReader()); + var factory = new MockClientFactory(); + var node = new ScriptedNodeConnector + { + ConnectAction = (n, _) => + { + n.SimulateStatus(ConnectionStatus.Connected); + n.SimulatePairing(PairingStatus.Paired); + } + }; + using var manager = new GatewayConnectionManager( + resolver, factory, _registry, NullLogger.Instance, + nodeConnector: node, + isNodeEnabled: () => true, + reconnectDelay: _ => Task.CompletedTask); + + await manager.ConnectAsync("gw-local"); + await InvokeHandshakeSucceededAsync(manager); + await WaitUntilAsync(() => node.ConnectCount >= 1); + var before = node.ConnectCount; + + // A wrong shared token / generic auth is NOT a device-token mismatch: never clear the token. + node.SimulateConnectionFailure(GatewayErrorKind.Auth); + await Task.Delay(150); + + Assert.Equal("stale-node-token", DeviceIdentity.TryReadStoredDeviceTokenForRole(identityDir, "node")); + Assert.Equal(before, node.ConnectCount); + } + + [Fact] + public async Task ReconnectIfCurrentAsync_GatewayNotActive_ReturnsFalseWithoutConnecting() + { + SetupGateway("gw-1", "wss://one"); + _registry.AddOrUpdate(new GatewayRecord { Id = "gw-2", Url = "wss://two" }); + _registry.SetActive("gw-2"); + _resolver.OperatorCredential = new GatewayCredential("tok", false, "test"); + + // Auto-repair pinned to gw-1, but gw-2 is active: must no-op, never connect gw-1. + var reconnected = await _manager.ReconnectIfCurrentAsync("gw-1"); + + Assert.False(reconnected); + Assert.DoesNotContain("wss://one", _factory.CreatedGatewayUrls); + } + + [Fact] + public async Task ReconnectIfCurrentAsync_GatewayActive_ReconnectsSameGateway() + { + SetupGateway("gw-1", "wss://one"); + _resolver.OperatorCredential = new GatewayCredential("tok", false, "test"); + await _manager.ConnectAsync("gw-1"); + var createdBefore = _factory.CreatedClients.Count; + + var reconnected = await _manager.ReconnectIfCurrentAsync("gw-1"); + + Assert.True(reconnected); + Assert.True(_factory.CreatedClients.Count > createdBefore); // fresh connect for the same gateway + Assert.Equal("gw-1", _registry.ActiveGatewayId); + } + + [Fact] + public async Task ReconnectIfCurrentAsync_CredentialResolutionFails_ReturnsFalse() + { + SetupGateway("gw-1", "wss://one"); + _resolver.OperatorCredential = null; // ConnectCoreAsync bails to Error without creating a client + + var reconnected = await _manager.ReconnectIfCurrentAsync("gw-1"); + + // Must NOT report success for a credential failure, or auto-repair would restart WSL to "fix" it. + Assert.False(reconnected); + } + + [Fact] + public async Task UserDisconnectIntent_BlocksAutomaticReconnect_UntilExplicitConnect() + { + SetupGateway("gw-1", "wss://one"); + _resolver.OperatorCredential = new GatewayCredential("tok", false, "test"); + await _manager.ConnectAsync("gw-1"); + + await _manager.DisconnectByUserAsync(); + + Assert.False(_manager.IsAutomaticReconnectAllowed("gw-1")); + Assert.False(await _manager.ReconnectIfCurrentAsync("gw-1")); + + await _manager.ConnectAsync("gw-1"); + Assert.True(_manager.IsAutomaticReconnectAllowed("gw-1")); + } + + [Fact] + public async Task GatewayLifecycleLease_IsMutuallyExclusive_ManualVsAuto() + { + Assert.False(_manager.IsManualGatewayLifecycleInProgress); + + // Manual op acquires the shared lease and marks itself as a manual holder. + var manual = await _manager.BeginManualGatewayLifecycleOperationAsync(); + Assert.True(_manager.IsManualGatewayLifecycleInProgress); + + // Auto-repair's non-blocking acquire must fail while the manual op holds the lease. + Assert.Null(_manager.TryAcquireGatewayLifecycleLease()); + + manual.Dispose(); + Assert.False(_manager.IsManualGatewayLifecycleInProgress); + + // Auto acquire now succeeds — and does NOT mark a manual holder (so the monitor is not falsely + // suppressed by an auto-repair's own restart). + var auto = _manager.TryAcquireGatewayLifecycleLease(); + Assert.NotNull(auto); + Assert.False(_manager.IsManualGatewayLifecycleInProgress); + + // A second concurrent acquire fails while the first is held. + Assert.Null(_manager.TryAcquireGatewayLifecycleLease()); + + auto!.Dispose(); + auto.Dispose(); // idempotent — must not over-release + Assert.NotNull(_manager.TryAcquireGatewayLifecycleLease()); + } + [Fact] public async Task HandshakeSucceeded_StartsNodeConnectorWithPersistedV2Requirement() { @@ -2213,6 +2802,9 @@ public void SimulateStatusChanged(ConnectionStatus status) => public void SimulateAuthFailed(string msg) => AuthenticationFailed?.Invoke(this, msg); + public void SimulateConnectionFailure(GatewayErrorKind kind) => + _client.SimulateConnectionFailure(kind); + public void SimulateTransportConnected() => _client.SimulateTransportConnected(); @@ -2236,6 +2828,9 @@ public MockGatewayClient(string url) public void SimulateTransportConnected() => RaiseTransportConnected(); + public void SimulateConnectionFailure(GatewayErrorKind kind) => + RaiseConnectionFailure(kind); + /// Simulate a successful hello-ok handshake for testing. public void SimulateHandshakeSucceeded() { @@ -2736,6 +3331,7 @@ private sealed class ScriptedNodeConnector : INodeConnector, INodeConnectorTelem { public int ConnectCount { get; private set; } public string? LastGatewayUrl { get; private set; } + public GatewayCredential? LastCredential { get; private set; } public bool IsConnected { get; private set; } public PairingStatus PairingStatus { get; private set; } = PairingStatus.Unknown; public string? NodeDeviceId => "scripted-node"; @@ -2762,6 +3358,7 @@ public Task ConnectAsync(string gatewayUrl, GatewayCredential credential, string { ConnectCount++; LastGatewayUrl = gatewayUrl; + LastCredential = credential; ConnectAction?.Invoke(this, gatewayUrl); return Task.CompletedTask; } diff --git a/tests/OpenClaw.Connection.Tests/GatewayRegistryMigrationTests.cs b/tests/OpenClaw.Connection.Tests/GatewayRegistryMigrationTests.cs index e4cc9c56f..280761773 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayRegistryMigrationTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayRegistryMigrationTests.cs @@ -49,6 +49,210 @@ public void MigrateFromSettings_WithBootstrapToken() Assert.Null(record.SharedGatewayToken); } + [Fact] + public void MigrateFromSettings_LocalhostWithSetupState_BackfillsManagedDistro() + { + var localRoot = Path.Combine(_tempDir, "local-root"); + var stateDir = Path.Combine(localRoot, "OpenClawTray"); + Directory.CreateDirectory(stateDir); + File.WriteAllText( + Path.Combine(stateDir, "setup-state.json"), + """{"DistroName":"OpenClawGateway","GatewayUrl":"ws://127.0.0.1:18789"}"""); + var previous = Environment.GetEnvironmentVariable("OPENCLAW_TRAY_LOCALAPPDATA_DIR"); + try + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCALAPPDATA_DIR", localRoot); + + Assert.True(_registry.MigrateFromSettings( + "ws://localhost:18789", + "shared-token", + null, + false, + null, + null, + 0, + 0, + _tempDir)); + + var record = _registry.GetActive()!; + Assert.Equal("OpenClawGateway", record.SetupManagedDistroName); + Assert.Equal("Local (OpenClawGateway)", record.FriendlyName); + } + finally + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCALAPPDATA_DIR", previous); + } + } + + [Fact] + public void MigrateFromSettings_DirectLocalDataOverride_BackfillsManagedDistro() + { + var direct = Path.Combine(_tempDir, "direct-setup-data"); + Directory.CreateDirectory(direct); + File.WriteAllText( + Path.Combine(direct, "setup-state.json"), + """{"DistroName":"OpenClawGateway-Dev","GatewayUrl":"ws://localhost:18789"}"""); + var previous = Environment.GetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR"); + try + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR", direct); + Assert.True(_registry.MigrateFromSettings( + "ws://localhost:18789", + "shared-token", + null, + false, + null, + null, + 0, + 0, + _tempDir)); + Assert.Equal( + "OpenClawGateway-Dev", + _registry.GetActive()!.SetupManagedDistroName); + } + finally + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR", previous); + } + } + + [Fact] + public void MigrateFromSettings_ManualLocalhostWithDifferentSetupEndpoint_RemainsManual() + { + var direct = Path.Combine(_tempDir, "manual-localhost-setup-data"); + Directory.CreateDirectory(direct); + File.WriteAllText( + Path.Combine(direct, "setup-state.json"), + """{"DistroName":"OpenClawGateway","GatewayUrl":"ws://localhost:18789"}"""); + var previous = Environment.GetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR"); + try + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR", direct); + + Assert.True(_registry.MigrateFromSettings( + "ws://localhost:19999", + "shared-token", + null, + false, + null, + null, + 0, + 0, + _tempDir)); + + var record = _registry.GetActive()!; + Assert.True(record.IsLocal); + Assert.Null(record.SetupManagedDistroName); + Assert.Null(record.FriendlyName); + } + finally + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR", previous); + } + } + + [Fact] + public void MigrateFromSettings_NonstandardLoopbackAliasWithSetupState_RemainsManual() + { + var direct = Path.Combine(_tempDir, "loopback-alias-setup-data"); + Directory.CreateDirectory(direct); + File.WriteAllText( + Path.Combine(direct, "setup-state.json"), + """{"DistroName":"OpenClawGateway","GatewayUrl":"ws://localhost:18789"}"""); + var previous = Environment.GetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR"); + try + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR", direct); + + Assert.True(_registry.MigrateFromSettings( + "ws://127.0.0.2:18789", + "shared-token", + null, + false, + null, + null, + 0, + 0, + _tempDir)); + + Assert.Null(_registry.GetActive()!.SetupManagedDistroName); + } + finally + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR", previous); + } + } + + [Fact] + public void MigrateFromSettings_SetupStateWithoutGatewayUrl_RemainsManual() + { + var direct = Path.Combine(_tempDir, "url-less-setup-data"); + Directory.CreateDirectory(direct); + File.WriteAllText( + Path.Combine(direct, "setup-state.json"), + """{"DistroName":"OpenClawGateway"}"""); + var previous = Environment.GetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR"); + try + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR", direct); + + Assert.True(_registry.MigrateFromSettings( + "ws://localhost:18789", + "shared-token", + null, + false, + null, + null, + 0, + 0, + _tempDir)); + + Assert.Null(_registry.GetActive()!.SetupManagedDistroName); + } + finally + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR", previous); + } + } + + [Fact] + public void MigrateFromSettings_DevSettingsPreferDevSetupState() + { + var localRoot = Path.Combine(_tempDir, "side-by-side-local"); + Directory.CreateDirectory(Path.Combine(localRoot, "OpenClawTray")); + Directory.CreateDirectory(Path.Combine(localRoot, "OpenClawTray-Dev")); + File.WriteAllText( + Path.Combine(localRoot, "OpenClawTray", "setup-state.json"), + """{"DistroName":"OpenClawGateway","GatewayUrl":"ws://localhost:18789"}"""); + File.WriteAllText( + Path.Combine(localRoot, "OpenClawTray-Dev", "setup-state.json"), + """{"DistroName":"OpenClawGateway-Dev","GatewayUrl":"ws://localhost:18790"}"""); + var previous = Environment.GetEnvironmentVariable("OPENCLAW_TRAY_LOCALAPPDATA_DIR"); + try + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCALAPPDATA_DIR", localRoot); + var devSettingsDir = Path.Combine(_tempDir, "OpenClawTray-Dev"); + Directory.CreateDirectory(devSettingsDir); + Assert.True(_registry.MigrateFromSettings( + "ws://localhost:18790", + "shared-token", + null, + false, + null, + null, + 0, + 0, + devSettingsDir)); + Assert.Equal( + "OpenClawGateway-Dev", + _registry.GetActive()!.SetupManagedDistroName); + } + finally + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCALAPPDATA_DIR", previous); + } + } + [Fact] public void MigrateFromSettings_WithSshTunnel() { @@ -63,6 +267,41 @@ public void MigrateFromSettings_WithSshTunnel() Assert.Equal("host.com", record.SshTunnel.Host); } + [Fact] + public void MigrateFromSettings_LocalSshTunnelWithMatchingSetupState_RemainsManual() + { + var direct = Path.Combine(_tempDir, "ssh-setup-data"); + Directory.CreateDirectory(direct); + File.WriteAllText( + Path.Combine(direct, "setup-state.json"), + """{"DistroName":"OpenClawGateway","GatewayUrl":"ws://localhost:18789"}"""); + var previous = Environment.GetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR"); + try + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR", direct); + + Assert.True(_registry.MigrateFromSettings( + "ws://localhost:18789", + "shared-token", + null, + true, + "user", + "host.example", + 22, + 18789, + _tempDir)); + + var record = _registry.GetActive()!; + Assert.NotNull(record.SshTunnel); + Assert.Null(record.SetupManagedDistroName); + Assert.Null(record.FriendlyName); + } + finally + { + Environment.SetEnvironmentVariable("OPENCLAW_TRAY_LOCAL_DATA_DIR", previous); + } + } + [Fact] public void MigrateFromSettings_WithSshTunnelBrowserProxyForward_PreservesFlag() { diff --git a/tests/OpenClaw.Connection.Tests/GatewayRegistryTests.cs b/tests/OpenClaw.Connection.Tests/GatewayRegistryTests.cs index 68518e74f..08c034e9f 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayRegistryTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayRegistryTests.cs @@ -437,6 +437,216 @@ public void PreserveAdvancedFields_FormValueWins_AndNullExistingIsNoOp() Assert.Same(fresh, preserved); } + [Fact] + public void PreserveAdvancedFields_KeepsManagedLocalOwnership_WhenEditedEndpointStaysLoopback() + { + // A setup-managed local gateway edited (name/token) but still pointing at loopback must keep + // its managed-local ownership so keepalive + auto-repair keep working across the edit. + var existing = MakeRecord("gw-1", "ws://localhost:18789") with + { + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + RequiresV2Signature = true, + }; + + var rebuilt = new GatewayRecord + { + Id = "gw-1", + Url = "ws://localhost:18789", + FriendlyName = "renamed", + SharedGatewayToken = "rotated", + }.PreserveAdvancedFields(existing); + + Assert.True(rebuilt.IsLocal); + Assert.Equal("OpenClawGateway", rebuilt.SetupManagedDistroName); + Assert.True(rebuilt.RequiresV2Signature); + } + + [Fact] + public void PreserveAdvancedFields_KeepsManagedOwnership_ForNonLoopbackManagedGateway_WhenUrlUnchanged() + { + // A setup-managed gateway can have a NON-loopback URL (e.g. Tailscale wss://…ts.net) while + // still being IsLocal + WSL-managed. Editing only its name/token (URL unchanged) must keep its + // managed ownership so keepalive + auto-repair keep working. + var existing = MakeRecord("gw-ts", "wss://host.tailnet.ts.net") with + { + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + }; + + var rebuilt = new GatewayRecord + { + Id = "gw-ts", + Url = "wss://host.tailnet.ts.net", // unchanged + FriendlyName = "renamed", + SharedGatewayToken = "rotated", + }.PreserveAdvancedFields(existing); + + Assert.True(rebuilt.IsLocal); + Assert.Equal("OpenClawGateway", rebuilt.SetupManagedDistroName); + } + + [Fact] + public void PreserveAdvancedFields_MigratesLegacyManagedRecord_ToExplicitDistroName() + { + // Legacy setup-managed records carry IsLocal=true but NO SetupManagedDistroName (managed status + // is recognized downstream via the default friendly name). A token-only edit migrates that + // implicit ownership to the explicit durable distro marker. + var existing = MakeRecord("gw-legacy", "ws://localhost:18789") with + { + IsLocal = true, + FriendlyName = "Local (OpenClawGateway)" + }; + + var rebuilt = new GatewayRecord + { + Id = "gw-legacy", + Url = "ws://localhost:18789", + SharedGatewayToken = "rotated", + }.PreserveAdvancedFields(existing); + + Assert.True(rebuilt.IsLocal); + Assert.Equal("OpenClawGateway", rebuilt.SetupManagedDistroName); + } + + [Fact] + public void PreserveAdvancedFields_ChangedLoopbackPort_DropsManagedOwnership() + { + var existing = MakeRecord("gw-1", "ws://localhost:18789") with + { + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + FriendlyName = "Local (OpenClawGateway)", + RequiresV2Signature = true, + }; + + var rebuilt = new GatewayRecord + { + Id = "gw-1", + Url = "ws://127.0.0.1:18800", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + FriendlyName = "Local ( OpenClawGateway )", + RequiresV2Signature = true, + }.PreserveAdvancedFields(existing); + + Assert.True(rebuilt.IsLocal); + Assert.Null(rebuilt.SetupManagedDistroName); + Assert.Null(rebuilt.FriendlyName); + Assert.False(rebuilt.RequiresV2Signature); + Assert.Null(GatewayRecordEditing.ResolveManagedDistroName(rebuilt)); + } + + [Fact] + public void PreserveAdvancedFields_DifferentLoopbackHost_DropsManagedOwnership() + { + var existing = MakeRecord("gw-1", "ws://localhost:18789") with + { + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + FriendlyName = "Local (OpenClawGateway)", + }; + + var rebuilt = new GatewayRecord + { + Id = "gw-1", + Url = "ws://127.0.0.2:18789", + IsLocal = true, + FriendlyName = "Local (OpenClawGateway)", + }.PreserveAdvancedFields(existing); + + Assert.False(rebuilt.IsLocal); + Assert.Null(rebuilt.SetupManagedDistroName); + Assert.Null(rebuilt.FriendlyName); + Assert.Null(GatewayRecordEditing.ResolveManagedDistroName(rebuilt)); + } + + [Theory] + [InlineData("ws://127.0.0.1:18789/")] + [InlineData("ws://[::1]:18789/")] + public void PreserveAdvancedFields_EquivalentLoopbackAlias_KeepsManagedOwnership(string editedUrl) + { + var existing = MakeRecord("gw-1", "ws://localhost:18789/") with + { + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + }; + + var rebuilt = new GatewayRecord + { + Id = "gw-1", + Url = editedUrl, + }.PreserveAdvancedFields(existing); + + Assert.Equal("OpenClawGateway", rebuilt.SetupManagedDistroName); + } + + [Fact] + public void PreserveAdvancedFields_PathOrQueryCaseChange_DropsManagedOwnership() + { + var existing = MakeRecord("gw-1", "ws://localhost:18789/Case?Token=A") with + { + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + FriendlyName = "Local (OpenClawGateway)", + }; + + var rebuilt = new GatewayRecord + { + Id = "gw-1", + Url = "ws://LOCALHOST:18789/case?token=a", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + FriendlyName = "Local (OpenClawGateway)", + }.PreserveAdvancedFields(existing); + + Assert.True(rebuilt.IsLocal); + Assert.Null(rebuilt.SetupManagedDistroName); + Assert.Null(rebuilt.FriendlyName); + Assert.Null(GatewayRecordEditing.ResolveManagedDistroName(rebuilt)); + } + + [Fact] + public void PreserveAdvancedFields_DropsManagedLocalOwnership_WhenRepointedToRemote() + { + // If the user repoints a managed-local gateway at a remote host, it is no longer a managed WSL + // gateway — the ownership fields must NOT be carried (auto-repair must not restart a WSL distro + // for a remote endpoint). + var existing = MakeRecord("gw-1", "ws://localhost:18789") with + { + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + }; + + var rebuilt = new GatewayRecord + { + Id = "gw-1", + Url = "wss://remote.example:443", + }.PreserveAdvancedFields(existing); + + Assert.False(rebuilt.IsLocal); + Assert.Null(rebuilt.SetupManagedDistroName); + } + + [Fact] + public void PreserveAdvancedFields_DropsManagedLocalOwnership_WhenSshTunnelAdded() + { + var existing = MakeRecord("gw-1", "ws://localhost:18789") with + { + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + }; + + var rebuilt = new GatewayRecord + { + Id = "gw-1", + Url = "ws://localhost:18789", + SshTunnel = new SshTunnelConfig("user", "host.example", 18789, 45678), + }.PreserveAdvancedFields(existing); + + Assert.Null(rebuilt.SetupManagedDistroName); + } + private static GatewayRecord MakeRecord(string id, string url) => new() { Id = id, diff --git a/tests/OpenClaw.Connection.Tests/InteractiveGatewayCredentialResolverTests.cs b/tests/OpenClaw.Connection.Tests/InteractiveGatewayCredentialResolverTests.cs index b371874bc..78757c2c2 100644 --- a/tests/OpenClaw.Connection.Tests/InteractiveGatewayCredentialResolverTests.cs +++ b/tests/OpenClaw.Connection.Tests/InteractiveGatewayCredentialResolverTests.cs @@ -52,6 +52,51 @@ public void TryResolve_UsesActiveGatewaySharedToken() Assert.Equal(CredentialResolver.SourceSharedGatewayToken, credential.Source); } + [Fact] + public void TryResolve_ManagedLoopbackUnknownOwner_DoesNotReturnTokenBearingCredential() + { + var record = new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + SharedGatewayToken = "shared-token" + }; + _registry.AddOrUpdate(record); + _registry.SetActive(record.Id); + + var resolved = InteractiveGatewayCredentialResolver.TryResolve( + _registry, + _tempDir, + _identityReader, + record.Url, + null, + null, + authorizeCredential: (_, _) => false, + out var credential); + + Assert.False(resolved); + Assert.Null(credential); + } + + [Fact] + public void TryResolve_LegacyLoopbackUnknownOwner_DoesNotReturnTokenBearingCredential() + { + var resolved = InteractiveGatewayCredentialResolver.TryResolve( + registry: null, + settingsDirectory: _tempDir, + identityReader: _identityReader, + effectiveGatewayUrl: "ws://localhost:18789", + legacyToken: "legacy-shared-token", + legacyBootstrapToken: null, + authorizeCredential: (_, _) => false, + out var credential); + + Assert.False(resolved); + Assert.Null(credential); + } + [Fact] public void TryResolve_PreservesBootstrapPairingState() { diff --git a/tests/OpenClaw.Connection.Tests/ManagedLocalGatewayPortProvenanceServiceTests.cs b/tests/OpenClaw.Connection.Tests/ManagedLocalGatewayPortProvenanceServiceTests.cs new file mode 100644 index 000000000..b073bdce9 --- /dev/null +++ b/tests/OpenClaw.Connection.Tests/ManagedLocalGatewayPortProvenanceServiceTests.cs @@ -0,0 +1,471 @@ +using System.Net; +using OpenClaw.Shared; +using Xunit; + +namespace OpenClaw.Connection.Tests; + +public class ManagedLocalGatewayPortProvenanceServiceTests +{ + private static GatewayRecord ManagedRecord() => new() + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + }; + + [Fact] + public void CreateWslRelaySignatureProbe_DoesNotInheritPwshModulePath() + { + const string relayPath = @"C:\Program Files\WSL\wslrelay.exe"; + + var startInfo = + WindowsManagedLocalGatewayPortPlatform.CreateWslRelaySignatureProbe(relayPath); + + Assert.False(startInfo.Environment.ContainsKey("PSModulePath")); + Assert.Equal(relayPath, startInfo.Environment["OPENCLAW_VERIFY_PATH"]); + Assert.Contains( + "Get-AuthenticodeSignature", + startInfo.ArgumentList[^1], + StringComparison.Ordinal); + } + + [Fact] + public void CreateExpectedWslGatewayProbe_UsesStdinAndDoesNotAssumeRelayFamilyParity() + { + var probe = WindowsManagedLocalGatewayPortPlatform.CreateExpectedWslGatewayProbe( + "OpenClawE2E-test", + 18789); + + Assert.True(probe.StartInfo.RedirectStandardInput); + Assert.Equal( + ["-d", "OpenClawE2E-test", "--", "bash", "-s"], + probe.StartInfo.ArgumentList); + Assert.Contains("pid=$(systemctl", probe.StandardInput, StringComparison.Ordinal); + Assert.Contains("ss -ltnp", probe.StandardInput, StringComparison.Ordinal); + Assert.DoesNotContain("ss -4", probe.StandardInput, StringComparison.Ordinal); + Assert.DoesNotContain("ss -6", probe.StandardInput, StringComparison.Ordinal); + Assert.Contains("pid=$pid,", probe.StandardInput, StringComparison.Ordinal); + Assert.DoesNotContain("$pid", string.Join(" ", probe.StartInfo.ArgumentList), StringComparison.Ordinal); + } + + [Fact] + public void Inspect_WslRelayOnTargetAddress_IsExpectedManagedGateway() + { + var platform = new FakePlatform(); + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, + 18789, + 100, + "wslrelay", + @"C:\Program Files\WSL\wslrelay.exe")); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + + var result = service.Inspect(ManagedRecord()); + + Assert.Equal(GatewayEndpointProvenanceKind.ExpectedManagedGateway, result.Kind); + } + + [Fact] + public void Inspect_DualStackWslRelay_VerifiesExternalProofsOnce() + { + var platform = new FakePlatform(); + var start = new DateTime(2026, 7, 24, 1, 0, 0, DateTimeKind.Utc); + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, + 18789, + 100, + "wslrelay", + @"C:\Program Files\WSL\wslrelay.exe", + start)); + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.IPv6Loopback, + 18789, + 100, + "wslrelay", + @"C:\Program Files\WSL\wslrelay.exe", + start)); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + + var result = service.Inspect(ManagedRecord()); + + Assert.Equal(GatewayEndpointProvenanceKind.ExpectedManagedGateway, result.Kind); + Assert.Equal(1, platform.TrustedWslRelayChecks); + Assert.Equal(1, platform.ExpectedDistroChecks); + } + + [Fact] + public void Inspect_SpoofedWslRelayPath_IsUnknown() + { + var platform = new FakePlatform { TrustedWslRelay = false }; + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, + 18789, + 101, + "wslrelay", + @"C:\Temp\WSL\wslrelay.exe")); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + + var result = service.Inspect(ManagedRecord()); + + Assert.Equal(GatewayEndpointProvenanceKind.UnknownListener, result.Kind); + Assert.Contains("not the canonical Microsoft-signed binary", result.Detail); + } + + [Fact] + public void Inspect_RelayForDifferentOrInactiveDistro_IsUnknown() + { + var platform = new FakePlatform { ExpectedDistroListening = false }; + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, + 18789, + 102, + "wslrelay", + @"C:\Program Files\WSL\wslrelay.exe")); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + + var result = service.Inspect(ManagedRecord()); + + Assert.Equal(GatewayEndpointProvenanceKind.UnknownListener, result.Kind); + Assert.Contains("does not report its systemd gateway MainPID owning port", result.Detail); + } + + [Fact] + public void Inspect_LocalhostWithIncompleteIpv6Capture_FailsClosed() + { + var platform = new FakePlatform { Ipv6Complete = false }; + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, + 18789, + 102, + "wslrelay", + @"C:\Program Files\WSL\wslrelay.exe", + new DateTime(2026, 7, 24, 1, 0, 0, DateTimeKind.Utc))); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + + Assert.Equal( + GatewayEndpointProvenanceKind.UnknownListener, + service.Inspect(ManagedRecord()).Kind); + } + + [Fact] + public void Inspect_OwnerChangesDuringSlowVerification_FailsClosed() + { + var platform = new FakePlatform { ReplaceOwnerOnSecondCapture = true }; + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, + 18789, + 102, + "wslrelay", + @"C:\Program Files\WSL\wslrelay.exe", + new DateTime(2026, 7, 24, 1, 0, 0, DateTimeKind.Utc))); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + + Assert.Equal( + GatewayEndpointProvenanceKind.UnknownListener, + service.Inspect(ManagedRecord()).Kind); + } + + [Fact] + public void InteractiveCredentialGate_ExpectedCacheThenOwnerChanges_FailsClosed() + { + var platform = new FakePlatform(); + var start = new DateTime(2026, 7, 24, 1, 0, 0, DateTimeKind.Utc); + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, + 18789, + 102, + "wslrelay", + @"C:\Program Files\WSL\wslrelay.exe", + start)); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + var record = ManagedRecord(); + var credential = new GatewayCredential( + "shared-token", + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken); + Assert.Equal( + GatewayEndpointProvenanceKind.ExpectedManagedGateway, + service.Inspect(record).Kind); + Assert.True(service.IsStrongCredentialAllowed(record, credential)); + + platform.Listeners[0] = new WindowsTcpListenerInfo( + IPAddress.Loopback, + 18789, + 999, + "unknown", + @"C:\unknown.exe", + start.AddSeconds(1)); + + Assert.False(service.IsStrongCredentialAllowed(record, credential)); + } + + [Fact] + public void InteractiveCredentialGate_BrowserControlInspectDoesNotEvictGatewayProof() + { + var platform = new FakePlatform(); + var start = new DateTime(2026, 7, 24, 1, 0, 0, DateTimeKind.Utc); + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, 18789, 102, "wslrelay", + @"C:\Program Files\WSL\wslrelay.exe", start)); + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, 18791, 102, "wslrelay", + @"C:\Program Files\WSL\wslrelay.exe", start)); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + var gateway = ManagedRecord(); + var control = gateway with { Url = "ws://localhost:18791" }; + var credential = new GatewayCredential( + "shared-token", false, CredentialResolver.SourceSharedGatewayToken); + + Assert.Equal(GatewayEndpointProvenanceKind.ExpectedManagedGateway, service.Inspect(gateway).Kind); + Assert.Equal(GatewayEndpointProvenanceKind.ExpectedManagedGateway, service.Inspect(control).Kind); + Assert.True(service.IsStrongCredentialAllowed(gateway, credential)); + } + + [Fact] + public void InteractiveCredentialGate_GuestSocketOwnershipChanges_FailsClosed() + { + var platform = new FakePlatform(); + var start = new DateTime(2026, 7, 24, 1, 0, 0, DateTimeKind.Utc); + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, 18789, 102, "wslrelay", + @"C:\Program Files\WSL\wslrelay.exe", start)); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + var gateway = ManagedRecord(); + var credential = new GatewayCredential( + "shared-token", false, CredentialResolver.SourceSharedGatewayToken); + Assert.Equal(GatewayEndpointProvenanceKind.ExpectedManagedGateway, service.Inspect(gateway).Kind); + + platform.ExpectedDistroListening = false; + + Assert.False(service.IsStrongCredentialAllowed(gateway, credential)); + } + + [Fact] + public void Inspect_MixedRelevantOwners_IsUnknown() + { + var platform = new FakePlatform(); + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, 18789, 100, "wslrelay", @"C:\Program Files\WSL\wslrelay.exe")); + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.IPv6Loopback, 18789, 200, "other", @"C:\other.exe")); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + + var result = service.Inspect(ManagedRecord()); + + Assert.Equal(GatewayEndpointProvenanceKind.UnknownListener, result.Kind); + } + + [Fact] + public async Task RepairConflict_ProvenNativeOpenClaw_DisablesTaskThenStopsExactPid() + { + var platform = FakePlatform.WithProvenNativeGateway(); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + + var proof = service.Inspect(ManagedRecord()); + var result = await service.RepairConflictAsync(ManagedRecord(), CancellationToken.None); + + Assert.Equal(GatewayEndpointProvenanceKind.ConflictingOpenClawGateway, proof.Kind); + Assert.Equal(ManagedLocalPortConflictRepairOutcome.Repaired, result.Outcome); + Assert.Equal( + ["disable:OpenClaw Gateway (OpenClawGateway)", + "end:OpenClaw Gateway (OpenClawGateway)", + "stop:2144"], + platform.Actions); + } + + [Fact] + public async Task RepairConflict_TaskEndAlreadyReleasedPort_IsSuccessWithoutPidKill() + { + var platform = FakePlatform.WithProvenNativeGateway(); + platform.EndRemovesListener = true; + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + + var result = await service.RepairConflictAsync(ManagedRecord(), CancellationToken.None); + + Assert.Equal(ManagedLocalPortConflictRepairOutcome.Repaired, result.Outcome); + Assert.Equal( + ["disable:OpenClaw Gateway (OpenClawGateway)", + "end:OpenClaw Gateway (OpenClawGateway)"], + platform.Actions); + } + + [Fact] + public async Task RepairConflict_ReusedPidAfterTaskEnd_IsNeverKilled() + { + var platform = FakePlatform.WithProvenNativeGateway(); + platform.ReplaceProcessIdentityOnEnd = true; + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + + var result = await service.RepairConflictAsync(ManagedRecord(), CancellationToken.None); + + Assert.Equal(ManagedLocalPortConflictRepairOutcome.BlockedUnknownOwner, result.Outcome); + Assert.DoesNotContain("stop:2144", platform.Actions); + } + + [Fact] + public async Task RepairConflict_UserIntentChangesBeforeDisable_NoDestructiveActionRuns() + { + var platform = FakePlatform.WithProvenNativeGateway(); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + var checks = 0; + + var result = await service.RepairConflictAsync( + ManagedRecord(), + CancellationToken.None, + canContinue: () => ++checks < 2); + + Assert.Equal(ManagedLocalPortConflictRepairOutcome.BlockedUnknownOwner, result.Outcome); + Assert.Empty(platform.Actions); + } + + [Fact] + public async Task RepairConflict_UnknownOwner_NeverDisablesOrStops() + { + var platform = new FakePlatform(); + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, 18789, 300, "unknown", @"C:\unknown.exe")); + var service = new ManagedLocalGatewayPortProvenanceService(platform, NullLogger.Instance); + + var result = await service.RepairConflictAsync(ManagedRecord(), CancellationToken.None); + + Assert.Equal(ManagedLocalPortConflictRepairOutcome.BlockedUnknownOwner, result.Outcome); + Assert.Empty(platform.Actions); + } + + [Fact] + public void Inspect_TaskXmlEscapedProfilePath_IsStillProven() + { + const string profileRoot = @"C:\Users\A & B"; + var platform = FakePlatform.WithProvenNativeGateway(profileRoot); + var service = new ManagedLocalGatewayPortProvenanceService( + platform, + NullLogger.Instance, + () => profileRoot); + + Assert.Equal( + GatewayEndpointProvenanceKind.ConflictingOpenClawGateway, + service.Inspect(ManagedRecord()).Kind); + } + + private sealed class FakePlatform : IManagedLocalGatewayPortPlatform + { + public List Listeners { get; } = []; + public Dictionary CommandLines { get; } = []; + public Dictionary Files { get; } = new(StringComparer.OrdinalIgnoreCase); + public string? TaskXml { get; set; } + public List Actions { get; } = []; + public bool TrustedWslRelay { get; set; } = true; + public bool ExpectedDistroListening { get; set; } = true; + public bool EndRemovesListener { get; set; } + public bool ReplaceProcessIdentityOnEnd { get; set; } + public bool ReplaceOwnerOnSecondCapture { get; set; } + public bool Ipv4Complete { get; set; } = true; + public bool Ipv6Complete { get; set; } = true; + public int TrustedWslRelayChecks { get; private set; } + public int ExpectedDistroChecks { get; private set; } + private int _captureCount; + + public static FakePlatform WithProvenNativeGateway(string? userProfilePath = null) + { + var platform = new FakePlatform(); + var profileDir = Path.Combine( + userProfilePath ?? + Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), + ".openclaw-OpenClawGateway"); + var vbsPath = Path.Combine(profileDir, "gateway.vbs"); + var cmdPath = Path.Combine(profileDir, "gateway.cmd"); + platform.Listeners.Add(new WindowsTcpListenerInfo( + IPAddress.Loopback, + 18789, + 2144, + "node", + @"C:\Program Files\nodejs\node.exe", + new DateTime(2026, 7, 24, 0, 0, 0, DateTimeKind.Utc))); + platform.CommandLines[2144] = + @"""C:\Program Files\nodejs\node.exe"" C:\Users\test\AppData\Local\OpenClawTray\native-cli\node_modules\openclaw\dist\index.js gateway --port 18789"; + platform.TaskXml = new System.Xml.Linq.XDocument( + new System.Xml.Linq.XElement( + "Task", + new System.Xml.Linq.XElement( + "Actions", + new System.Xml.Linq.XElement( + "Exec", + new System.Xml.Linq.XElement("Command", vbsPath))))) + .ToString(); + platform.Files[vbsPath] = $"Run \"{cmdPath}\""; + platform.Files[cmdPath] = $""" + set "OPENCLAW_STATE_DIR={profileDir}" + set "OPENCLAW_WINDOWS_TASK_NAME=OpenClaw Gateway (OpenClawGateway)" + set "OPENCLAW_GATEWAY_PORT=18789" + C:\Users\test\AppData\Local\OpenClawTray\native-cli\node_modules\openclaw\dist\index.js gateway --port 18789 + """; + return platform; + } + + public WindowsTcpListenerSnapshotResult CaptureListeners() + { + _captureCount++; + if (ReplaceOwnerOnSecondCapture && _captureCount == 2 && Listeners.Count > 0) + { + return new( + [ + Listeners[0] with + { + ProcessId = 999, + ProcessName = "unknown", + ProcessPath = @"C:\unknown.exe", + ProcessStartTimeUtc = Listeners[0].ProcessStartTimeUtc?.AddSeconds(1) + } + ], + Ipv4Complete, + Ipv6Complete); + } + return new(Listeners.ToArray(), Ipv4Complete, Ipv6Complete); + } + public string? GetProcessCommandLine(int processId) => + CommandLines.GetValueOrDefault(processId); + public bool IsTrustedWslRelayBinary(string processPath) + { + TrustedWslRelayChecks++; + return TrustedWslRelay; + } + + public bool IsExpectedWslGatewayListening(string distroName, int port) + { + ExpectedDistroChecks++; + return ExpectedDistroListening; + } + public string? ReadScheduledTaskXml(string taskName) => TaskXml; + public string? ReadFile(string path) => Files.GetValueOrDefault(path); + + public Task DisableScheduledTaskAsync(string taskName, CancellationToken cancellationToken) + { + Actions.Add($"disable:{taskName}"); + return Task.FromResult(true); + } + + public Task EndScheduledTaskAsync(string taskName, CancellationToken cancellationToken) + { + Actions.Add($"end:{taskName}"); + if (EndRemovesListener) + Listeners.Clear(); + else if (ReplaceProcessIdentityOnEnd && Listeners.Count > 0) + Listeners[0] = Listeners[0] with + { + ProcessStartTimeUtc = Listeners[0].ProcessStartTimeUtc?.AddMinutes(1) + }; + return Task.CompletedTask; + } + + public Task StopProcessAsync( + int processId, + DateTime? expectedStartTimeUtc, + CancellationToken cancellationToken) + { + Actions.Add($"stop:{processId}"); + Listeners.RemoveAll(listener => listener.ProcessId == processId); + return Task.FromResult(true); + } + } +} diff --git a/tests/OpenClaw.Connection.Tests/OperatorScopeHelperTests.cs b/tests/OpenClaw.Connection.Tests/OperatorScopeHelperTests.cs index ebc37aaf6..2915f2e4f 100644 --- a/tests/OpenClaw.Connection.Tests/OperatorScopeHelperTests.cs +++ b/tests/OpenClaw.Connection.Tests/OperatorScopeHelperTests.cs @@ -170,8 +170,16 @@ public void SimulateConnected() public Task ConnectAsync(string? gatewayId = null) => Task.CompletedTask; public Task ConnectNodeOnlyAsync(string? gatewayId = null) => Task.CompletedTask; public Task DisconnectAsync() => Task.CompletedTask; + public Task DisconnectByUserAsync() => Task.CompletedTask; public Task ReconnectAsync() => Task.CompletedTask; + public Task ReconnectIfCurrentAsync(string gatewayId, CancellationToken cancellationToken = default) => + Task.FromResult(true); public Task SwitchGatewayAsync(string gatewayId) => Task.CompletedTask; + public void SetGatewayConnectionIntent(string gatewayId, bool shouldBeConnected) { } + public bool IsAutomaticReconnectAllowed(string gatewayId) => true; + public bool IsManualGatewayLifecycleInProgress => false; + public Task BeginManualGatewayLifecycleOperationAsync(CancellationToken cancellationToken = default) => Task.FromResult(new NoopScope()); + private sealed class NoopScope : IDisposable { public void Dispose() { } } public Task EnsureNodeConnectedAsync(CancellationToken cancellationToken = default) => Task.CompletedTask; public Task ApplySetupCodeAsync(string setupCode, SshTunnelConfig? sshTunnel = null) => Task.FromResult(new SetupCodeResult(SetupCodeOutcome.InvalidCode)); public Task ConnectWithSharedTokenAsync(string gatewayUrl, string token, SshTunnelConfig? sshTunnel = null) => Task.FromResult(new SetupCodeResult(SetupCodeOutcome.InvalidCode)); diff --git a/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs b/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs index 507cfecc2..ef0d0c1ab 100644 --- a/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs +++ b/tests/OpenClaw.E2ETests/Setup/E2ESetupFixture.cs @@ -677,10 +677,11 @@ public string ReadActiveGatewayDeviceId() $"Timed out waiting for durable paired credentials. Last={last?.ToString() ?? ""}; error={lastError?.Message}"); } - public async Task WaitForTrayKeepAliveStartedAsync(TimeSpan? timeout = null) + public async Task WaitForTrayKeepAliveReadyAsync(TimeSpan? timeout = null) { var logPath = Path.Combine(DataDir, "openclaw-tray.log"); - var expected = $"[WslKeepAlive] Started keepalive for {_distroName}"; + var expectedStarted = $"[WslKeepAlive] Started keepalive for {_distroName}"; + var expectedExisting = $"[WslKeepAlive] Keepalive already running for {_distroName}"; var deadline = DateTime.UtcNow.Add(timeout ?? TimeSpan.FromSeconds(30)); string lastLogTail = ""; @@ -698,7 +699,9 @@ public async Task WaitForTrayKeepAliveStartedAsync(TimeSpan? timeout = n { var lines = await File.ReadAllLinesAsync(logPath); lastLogTail = string.Join(Environment.NewLine, lines.TakeLast(20)); - var match = lines.LastOrDefault(line => line.Contains(expected, StringComparison.Ordinal)); + var match = lines.LastOrDefault(line => + line.Contains(expectedStarted, StringComparison.Ordinal) || + line.Contains(expectedExisting, StringComparison.Ordinal)); if (match is not null) { Log($"Tray keepalive verified from log: {match}"); @@ -713,7 +716,8 @@ public async Task WaitForTrayKeepAliveStartedAsync(TimeSpan? timeout = n CopyDataDirLogs(); throw new TimeoutException( $"Tray did not log WSL keepalive startup within {timeout?.TotalSeconds ?? 30}s. " + - $"Expected: {expected}. Log tail: {lastLogTail}. Logs: {ArtifactDir}"); + $"Expected: {expectedStarted} or {expectedExisting}. " + + $"Log tail: {lastLogTail}. Logs: {ArtifactDir}"); } private static bool TryGetPropertyIgnoreCase(JsonElement element, string propertyName, out JsonElement value) diff --git a/tests/OpenClaw.E2ETests/Setup/SetupAndConnectTests.cs b/tests/OpenClaw.E2ETests/Setup/SetupAndConnectTests.cs index 2d8a07eae..4bfd71450 100644 --- a/tests/OpenClaw.E2ETests/Setup/SetupAndConnectTests.cs +++ b/tests/OpenClaw.E2ETests/Setup/SetupAndConnectTests.cs @@ -199,9 +199,9 @@ public async Task FullSetup_WindowsNodeContext_RemainsIdempotentAfterCrlfEdit() } [E2EFact] - public async Task FullSetup_TrayStartsWslKeepAlive() + public async Task FullSetup_TrayEnsuresWslKeepAlive() { - var logLine = await _fixture.WaitForTrayKeepAliveStartedAsync(); + var logLine = await _fixture.WaitForTrayKeepAliveReadyAsync(); Assert.Contains(_fixture.DistroName, logLine); var keepAlive = await _fixture.RunInWslAsync( diff --git a/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs b/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs index c7515c399..0e5af62b3 100644 --- a/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs +++ b/tests/OpenClaw.SetupEngine.Tests/SetupStepsTests.cs @@ -51,6 +51,29 @@ private SetupContext CreateContext(SetupConfig? config = null, ICommandRunner? c return new SetupContext(cfg, logger, journal, commands ?? new CommandRunner(logger), CancellationToken.None); } + [Fact] + public async Task PairingEndpointTrust_UnknownLoopbackOwner_BlocksBeforeCredentialUse() + { + var context = CreateContext(new SetupConfig + { + DistroName = "OpenClawGateway", + GatewayUrl = "ws://localhost:18789" + }); + context.EndpointProvenanceProbe = (_, _) => Task.FromResult( + new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + 18789, + Detail: "unknown owner")); + + var result = await PairOperatorStep.EnsurePairingEndpointTrustedAsync( + context, + CancellationToken.None); + + Assert.NotNull(result); + Assert.Equal(StepOutcome.FailedTerminal, result!.Outcome); + Assert.Contains("unknown owner", result.Message); + } + [Theory] [InlineData(null)] [InlineData("")] diff --git a/tests/OpenClaw.Shared.Tests/CapabilityTests.cs b/tests/OpenClaw.Shared.Tests/CapabilityTests.cs index 1bec6f133..70d828f2d 100644 --- a/tests/OpenClaw.Shared.Tests/CapabilityTests.cs +++ b/tests/OpenClaw.Shared.Tests/CapabilityTests.cs @@ -1230,6 +1230,28 @@ public async Task BrowserProxy_ForwardsToLocalControlPortWithBearerAuth() Assert.True(result.GetProperty("ok").GetBoolean()); } + [Fact] + public async Task BrowserProxy_UnverifiedControlListener_DoesNotSendBearerToken() + { + var handler = new CapturingHandler("""{"ok":true}"""); + var cap = new BrowserProxyCapability( + NullLogger.Instance, + "ws://127.0.0.1:18789", + "secret-token", + handler, + authorizeEndpointAsync: (_, _) => Task.FromResult(false)); + + var res = await cap.ExecuteAsync(new NodeInvokeRequest + { + Id = "browser-blocked", + Command = "browser.proxy", + Args = Parse("""{"method":"GET","path":"/snapshot"}""") + }); + + Assert.False(res.Ok); + Assert.Null(handler.LastRequest); + } + [Fact] public async Task BrowserProxy_RemoteGatewayWithoutOverride_DoesNotSendTokenToLocalFallback() { diff --git a/tests/OpenClaw.Shared.Tests/GatewayErrorClassifierTests.cs b/tests/OpenClaw.Shared.Tests/GatewayErrorClassifierTests.cs index b08636b03..1d73429b5 100644 --- a/tests/OpenClaw.Shared.Tests/GatewayErrorClassifierTests.cs +++ b/tests/OpenClaw.Shared.Tests/GatewayErrorClassifierTests.cs @@ -156,4 +156,95 @@ public void Classify_TokenDriftWins_OverGenericAuthKeywords() GatewayErrorKind.TokenDrift, GatewayErrorClassifier.Classify("auth failed: device token no longer valid, re-pair required")); } + + [Fact] + public void Classify_LiveGatewayTokenMismatch_IsAuth_NotDeviceTokenDrift() + { + const string error = + "unauthorized: gateway token mismatch (set gateway.remote.token to match gateway.auth.token)"; + + Assert.Equal(GatewayErrorKind.Auth, GatewayErrorClassifier.Classify(error)); + Assert.True(GatewayErrorClassifier.IsSharedGatewayTokenMismatch(error)); + } + + // ─── ClassifyWithCode: exact device-vs-shared token distinction ─── + + [Theory] + [InlineData("AUTH_DEVICE_TOKEN_MISMATCH", null)] + [InlineData(null, "AUTH_DEVICE_TOKEN_MISMATCH")] + [InlineData("auth_device_token_mismatch", null)] + public void ClassifyWithCode_DeviceTokenMismatchCode_IsDeviceTokenMismatch(string? topLevel, string? detailsCode) + { + // The structured device-token code is authoritative and must yield the exact + // auto-recoverable kind even when the human message is generic. + Assert.Equal( + GatewayErrorKind.DeviceTokenMismatch, + GatewayErrorClassifier.ClassifyWithCode("unauthorized", topLevel, detailsCode)); + } + + [Theory] + [InlineData("AUTH_TOKEN_MISMATCH")] + [InlineData("AUTH_BOOTSTRAP_TOKEN_INVALID")] + public void ClassifyWithCode_SharedOrBootstrapCode_IsNotDeviceTokenMismatch(string code) + { + // A wrong SHARED/gateway or bootstrap token must never be treated as a recoverable + // device-token drift — clearing the device token would just loop. + var kind = GatewayErrorClassifier.ClassifyWithCode("token mismatch", code, null); + Assert.NotEqual(GatewayErrorKind.DeviceTokenMismatch, kind); + } + + [Fact] + public void ClassifyWithCode_SharedTokenCodeEmbeddedInMessage_IsAuth() + { + Assert.Equal( + GatewayErrorKind.Auth, + GatewayErrorClassifier.ClassifyWithCode("AUTH_TOKEN_MISMATCH: unauthorized")); + } + + [Fact] + public void ClassifyWithCode_ExplicitDevicePhrase_IsDeviceTokenMismatch() + { + Assert.Equal( + GatewayErrorKind.DeviceTokenMismatch, + GatewayErrorClassifier.ClassifyWithCode("device token mismatch — rotate/reissue", null, null)); + } + + [Fact] + public void ClassifyWithCode_BareTokenMismatch_StaysBroadTokenDrift_NotDeviceExact() + { + // A bare "token mismatch" is device-vs-shared ambiguous: it must NOT be the exact + // auto-recoverable kind; it falls through to broad TokenDrift (manual re-pair). + Assert.Equal( + GatewayErrorKind.TokenDrift, + GatewayErrorClassifier.ClassifyWithCode("token mismatch", null, null)); + } + + [Fact] + public void ClassifyWithCode_NoCode_FallsBackToTextHeuristic() + { + Assert.Equal( + GatewayErrorKind.Network, + GatewayErrorClassifier.ClassifyWithCode("connection refused", null, null)); + } + + [Fact] + public void ClassifyWithCode_NonTokenStructuredCode_FoldsCodeIntoClassification() + { + // A non-token reason carried only in the structured code (with a generic "unauthorized" + // message) must keep the precision main had via Classify(code + " " + message) rather than + // dropping the code and mis-classifying as generic Auth. + Assert.Equal( + GatewayErrorKind.RateLimited, + GatewayErrorClassifier.ClassifyWithCode("unauthorized", "AUTH_RATE_LIMITED", null)); + } + + [Fact] + public void ClassifyWithCode_NonTokenCode_DoesNotOverrideDeviceMismatchCode() + { + // When both a device-token code and another code are present, the device code stays + // authoritative (checked first) and is not diluted by folding. + Assert.Equal( + GatewayErrorKind.DeviceTokenMismatch, + GatewayErrorClassifier.ClassifyWithCode("unauthorized", "AUTH_RATE_LIMITED", "AUTH_DEVICE_TOKEN_MISMATCH")); + } } diff --git a/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs b/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs index 1e8f3642b..7a619ca9f 100644 --- a/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs +++ b/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs @@ -495,6 +495,13 @@ public List CaptureAuthenticationFailedEvents() return events; } + public List CaptureConnectionFailures() + { + var events = new List(); + _client.ConnectionFailure += (_, kind) => events.Add(kind); + return events; + } + public (int WizardResponses, int RequestMethods) GetPendingRequestCounts() { var wizardField = typeof(OpenClawGatewayClient).GetField( @@ -3610,6 +3617,79 @@ public void HandleRequestError_TerminalAuthError_RaisesAuthenticationFailedEvent Assert.Contains("token mismatch", authEvents[0], StringComparison.OrdinalIgnoreCase); } + [Fact] + public void HandleRequestError_DeviceTokenMismatchTopLevelCode_GenericMessage_RaisesRecognizableAuthFailure() + { + // The gateway may deliver the device-token mismatch as a TOP-LEVEL error.code with a generic + // message. The raised AuthenticationFailed string must still be recognizable as a device-token + // mismatch so the connection manager can self-recover. + var helper = new GatewayClientTestHelper(); + var authEvents = helper.CaptureAuthenticationFailedEvents(); + helper.TrackPendingRequest("req-auth-toplevel", "connect"); + + helper.ProcessRawMessage(""" + { + "type": "res", + "id": "req-auth-toplevel", + "ok": false, + "error": { "message": "unauthorized", "code": "AUTH_DEVICE_TOKEN_MISMATCH" } + } + """); + + Assert.Single(authEvents); + Assert.Equal( + OpenClaw.Shared.GatewayErrorKind.DeviceTokenMismatch, + OpenClaw.Shared.GatewayErrorClassifier.ClassifyWithCode(authEvents[0])); + } + + [Fact] + public void HandleRequestError_SharedTokenMismatchTopLevelCode_DoesNotLookLikeDeviceMismatch() + { + // A wrong SHARED token (top-level AUTH_TOKEN_MISMATCH) is terminal auth but must NOT be + // enriched into a recoverable device-token mismatch. + var helper = new GatewayClientTestHelper(); + var authEvents = helper.CaptureAuthenticationFailedEvents(); + var failures = helper.CaptureConnectionFailures(); + helper.TrackPendingRequest("req-auth-shared", "connect"); + + helper.ProcessRawMessage(""" + { + "type": "res", + "id": "req-auth-shared", + "ok": false, + "error": { "message": "unauthorized", "code": "AUTH_TOKEN_MISMATCH" } + } + """); + + Assert.Single(authEvents); + Assert.Equal([GatewayErrorKind.Auth], failures); + Assert.NotEqual( + OpenClaw.Shared.GatewayErrorKind.DeviceTokenMismatch, + OpenClaw.Shared.GatewayErrorClassifier.ClassifyWithCode(authEvents[0])); + } + + [Fact] + public void HandleRequestError_SharedTokenMismatchNestedCode_RaisesTypedAuthFailure() + { + var helper = new GatewayClientTestHelper(); + var failures = helper.CaptureConnectionFailures(); + helper.TrackPendingRequest("req-auth-shared-nested", "connect"); + + helper.ProcessRawMessage(""" + { + "type": "res", + "id": "req-auth-shared-nested", + "ok": false, + "error": { + "message": "unauthorized", + "details": { "code": "AUTH_TOKEN_MISMATCH" } + } + } + """); + + Assert.Equal([GatewayErrorKind.Auth], failures); + } + [Fact] public void HandleRequestError_TerminalAuthError_RaisesErrorStatus() { diff --git a/tests/OpenClaw.Shared.Tests/WebSocketClientBaseTests.cs b/tests/OpenClaw.Shared.Tests/WebSocketClientBaseTests.cs index aeae55d13..874e6cc80 100644 --- a/tests/OpenClaw.Shared.Tests/WebSocketClientBaseTests.cs +++ b/tests/OpenClaw.Shared.Tests/WebSocketClientBaseTests.cs @@ -69,6 +69,7 @@ public void TestRaiseStatusChanged(ConnectionStatus status) public string TestGatewayUrlForDisplay => GatewayUrlForDisplay; public string TestToken => _token; public IOpenClawLogger TestLogger => _logger; + public Task TestReconnectWithBackoffAsync() => ReconnectWithBackoffAsync(); } [Collection("WebSocketClientBase")] @@ -97,6 +98,26 @@ public void Constructor_StoresToken() client.Dispose(); } + [Fact] + public async Task AutoReconnect_AuthorizationDenied_DoesNotOpenSocket() + { + using var client = new TestWebSocketClient("ws://127.0.0.1:1", "strong-token", _logger); + var authorizationCalls = 0; + client.ReconnectAuthorizationAsync = _ => + { + authorizationCalls++; + return Task.FromResult(new ReconnectAuthorizationResult( + false, + GatewayErrorKind.LocalPortConflict, + "blocked")); + }; + + await client.TestReconnectWithBackoffAsync(); + + Assert.Equal(1, authorizationCalls); + Assert.Equal(0, client.OnConnectedCallCount); + } + [Fact] public void Constructor_UsesNullLoggerWhenNotProvided() { @@ -155,9 +176,12 @@ public void Dispose_SetsIsDisposed() public void Dispose_IsIdempotent() { var client = new TestWebSocketClient("ws://localhost:18789", "token", _logger); + var disposedEvents = 0; + client.Disposed += (_, _) => disposedEvents++; client.Dispose(); client.Dispose(); // second call should not throw Assert.True(client.TestIsDisposed); + Assert.Equal(1, disposedEvents); Assert.Equal(1, client.OnDisposingCallCount); // hook called only once } diff --git a/tests/OpenClaw.Shared.Tests/WindowsNodeClientTests.cs b/tests/OpenClaw.Shared.Tests/WindowsNodeClientTests.cs index 2f0738742..80a973edf 100644 --- a/tests/OpenClaw.Shared.Tests/WindowsNodeClientTests.cs +++ b/tests/OpenClaw.Shared.Tests/WindowsNodeClientTests.cs @@ -107,7 +107,7 @@ public void Constructor_UsesAppVersionForRegistrationAndConnectMessage() [Theory] [InlineData("rate limit exceeded", GatewayErrorKind.RateLimited)] [InlineData("too many failed authentication attempts", GatewayErrorKind.RateLimited)] - [InlineData("device token mismatch", GatewayErrorKind.TokenDrift)] + [InlineData("device token mismatch", GatewayErrorKind.DeviceTokenMismatch)] [InlineData("origin not allowed", GatewayErrorKind.Auth)] [InlineData("gateway internal error", GatewayErrorKind.Server)] public void HandleResponse_TerminalError_EmitsFiniteFailureClassification( @@ -143,6 +143,78 @@ public void HandleResponse_TerminalError_EmitsFiniteFailureClassification( } } + [Theory] + [InlineData("AUTH_DEVICE_TOKEN_MISMATCH", "\"code\": \"AUTH_DEVICE_TOKEN_MISMATCH\"")] + [InlineData("details", "\"code\": \"none\", \"details\": { \"code\": \"AUTH_DEVICE_TOKEN_MISMATCH\" }")] + public void HandleResponse_NodeDeviceTokenMismatchByStructuredCode_GenericMessage_EmitsDeviceTokenMismatch( + string _, string codeJson) + { + // A stale node DEVICE token may arrive as a structured code (top-level error.code OR + // error.details.code) with a generic message. The node client must classify it as the exact + // DeviceTokenMismatch (so the manager can self-recover) regardless of message wording. + var dataPath = Path.Combine(Path.GetTempPath(), $"openclaw-node-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dataPath); + + try + { + using var client = new WindowsNodeClient("ws://localhost:18789", "test-token", dataPath); + GatewayErrorKind? actualKind = null; + client.ConnectionFailure += (_, kind) => actualKind = kind; + using var document = JsonDocument.Parse( + $$""" + { + "type": "res", + "ok": false, + "error": { "message": "unauthorized", {{codeJson}} } + } + """); + client.HandleResponse(document.RootElement); + + Assert.Equal(GatewayErrorKind.DeviceTokenMismatch, actualKind); + } + finally + { + Directory.Delete(dataPath, true); + } + } + + [Fact] + public void HandleResponse_SharedTokenMismatchByStructuredCode_GenericMessage_StopsReconnectAndIsNotDeviceMismatch() + { + // A wrong SHARED token delivered as a structured code (AUTH_TOKEN_MISMATCH) with a generic + // message is permanent for this connection: the node client must set _rateLimited to stop its + // OWN auto-reconnect loop, and must NOT classify it as a recoverable device-token mismatch. + var dataPath = Path.Combine(Path.GetTempPath(), $"openclaw-node-test-{Guid.NewGuid():N}"); + Directory.CreateDirectory(dataPath); + + try + { + using var client = new WindowsNodeClient("ws://localhost:18789", "test-token", dataPath); + GatewayErrorKind? actualKind = null; + client.ConnectionFailure += (_, kind) => actualKind = kind; + using var document = JsonDocument.Parse( + """ + { + "type": "res", + "ok": false, + "error": { "message": "unauthorized", "code": "AUTH_TOKEN_MISMATCH" } + } + """); + client.HandleResponse(document.RootElement); + + Assert.NotEqual(GatewayErrorKind.DeviceTokenMismatch, actualKind); + + var rateLimited = (bool)typeof(WindowsNodeClient) + .GetField("_rateLimited", BindingFlags.NonPublic | BindingFlags.Instance)! + .GetValue(client)!; + Assert.True(rateLimited); + } + finally + { + Directory.Delete(dataPath, true); + } + } + /// /// Regression test: when hello-ok includes auth.deviceToken, PairingStatusChanged must /// fire exactly once — not twice (once from the token block and again from the DeviceToken diff --git a/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs b/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs index 32a07e77d..2365775c8 100644 --- a/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs +++ b/tests/OpenClaw.Tray.Tests/AppRefactorContractTests.cs @@ -79,6 +79,34 @@ public void Startup_WslKeepAlive_IsOwnedByDedicatedService() Assert.Contains("ResolveLocalGatewayDistroNameAsync", service); } + [Fact] + public void ManagedLocalGatewayRepair_StaysDelegatedToDedicatedOwners() + { + var root = TestRepositoryPaths.GetRepositoryRoot(); + var app = ReadAppSources(); + var startup = ExtractMethod(app, "OnLaunchedAsync"); + var monitor = File.ReadAllText(Path.Combine( + root, + "src", + "OpenClaw.Tray.WinUI", + "Services", + "ManagedLocalGatewayAutoRepairMonitor.cs")); + var coordinator = File.ReadAllText(Path.Combine( + root, + "src", + "OpenClaw.Tray.WinUI", + "Services", + "ManagedLocalGatewayRepairCoordinator.cs")); + + Assert.Contains("new OpenClawTray.Services.ManagedLocalGatewayRepairCoordinator(", startup); + Assert.Contains("new OpenClawTray.Services.ManagedLocalGatewayAutoRepairMonitor(", startup); + Assert.Contains("private async Task RunAsync(CancellationToken cancellationToken)", monitor); + Assert.Contains("private async Task SafeProbeAsync", coordinator); + Assert.Contains("private async Task VerifyAsync", coordinator); + Assert.DoesNotContain("private async Task SafeProbeAsync", app); + Assert.DoesNotContain("private async Task VerifyAsync", app); + } + [Fact] public void McpOnlyStartup_DoesNotRequireGatewayCredentials() { diff --git a/tests/OpenClaw.Tray.Tests/ConnectionPagePlanApprovalBehaviorTests.cs b/tests/OpenClaw.Tray.Tests/ConnectionPagePlanApprovalBehaviorTests.cs index c6cbfc732..c5daff8c5 100644 --- a/tests/OpenClaw.Tray.Tests/ConnectionPagePlanApprovalBehaviorTests.cs +++ b/tests/OpenClaw.Tray.Tests/ConnectionPagePlanApprovalBehaviorTests.cs @@ -75,6 +75,45 @@ public void NodeListTrust_WithDiscoveryCommand_SuppressesGenericCommandWithoutCl Assert.False(plan.NodeTrustCommandApprovesRequest); } + [Fact] + public void SharedGatewayTokenMismatch_IsAuth_NotDeviceRePair() + { + var plan = Build( + new GatewayConnectionSnapshot + { + OverallState = OverallConnectionState.Error, + OperatorState = RoleConnectionState.Error, + OperatorError = + "unauthorized: gateway token mismatch (set gateway.remote.token to match gateway.auth.token)", + OperatorErrorKind = GatewayErrorKind.Auth, + GatewayUrl = "ws://localhost:18789", + }, + localNode: null); + + Assert.Equal(RecoveryCategory.Auth, plan.Recovery); + Assert.Equal("Authentication failed", plan.StripHeadline); + Assert.NotEqual("Device needs re-pairing", plan.StripHeadline); + } + + [Fact] + public void UnknownManagedLocalPortOwner_ShowsPortConflictRecovery() + { + var plan = Build( + new GatewayConnectionSnapshot + { + OverallState = OverallConnectionState.Error, + OperatorState = RoleConnectionState.Error, + OperatorError = "gateway token mismatch", + OperatorErrorKind = GatewayErrorKind.LocalPortConflict, + GatewayUrl = "ws://localhost:18789", + }, + localNode: null); + + Assert.Equal(RecoveryCategory.LocalPortConflict, plan.Recovery); + Assert.Equal("Local gateway port conflict", plan.StripHeadline); + Assert.Equal(ConnectionPrimaryAction.Retry, plan.StripPrimaryAction); + } + [Fact] public void NodeListTrust_OverridesNodeConnectingWaitState() { diff --git a/tests/OpenClaw.Tray.Tests/ManagedLocalGatewayAutoRepairMonitorTests.cs b/tests/OpenClaw.Tray.Tests/ManagedLocalGatewayAutoRepairMonitorTests.cs new file mode 100644 index 000000000..855e06091 --- /dev/null +++ b/tests/OpenClaw.Tray.Tests/ManagedLocalGatewayAutoRepairMonitorTests.cs @@ -0,0 +1,306 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using OpenClaw.Connection; +using OpenClaw.Shared; +using OpenClawTray.Services; +using Xunit; + +namespace OpenClaw.Tray.Tests; + +public class ManagedLocalGatewayAutoRepairMonitorTests : IDisposable +{ + private readonly string _tempDir; + private readonly GatewayRegistry _registry; + private readonly ConnectionDiagnostics _diagnostics = new(); + private readonly FakeClock _clock = new(); + private GatewayConnectionSnapshot _snapshot = GatewayConnectionSnapshot.Idle; + private bool _enabled = true; + private int _repairCount; + private int _resetBudgetCount; + private ManagedLocalGatewayRepairOutcome _repairOutcome = ManagedLocalGatewayRepairOutcome.Repaired; + + public ManagedLocalGatewayAutoRepairMonitorTests() + { + _tempDir = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "openclaw-autorepair-" + Guid.NewGuid().ToString("N")[..8]); + System.IO.Directory.CreateDirectory(_tempDir); + _registry = new GatewayRegistry(_tempDir); + } + + public void Dispose() + { + // slopwatch-ignore: SW003 Test cleanup is best-effort and must not hide the test outcome. + try { System.IO.Directory.Delete(_tempDir, true); } catch { } + } + + private void SetActiveManagedLocal() => + SetActive(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway" + }); + + private void SetActive(GatewayRecord record) + { + _registry.AddOrUpdate(record); + _registry.SetActive(record.Id); + } + + private ManagedLocalGatewayAutoRepairMonitor CreateMonitor( + TimeSpan? startupGrace = null, + Func? isAutomaticRepairAllowed = null) => + new( + () => _snapshot, + _registry, + _ => { _repairCount++; return Task.FromResult(new ManagedLocalGatewayRepairResult(_repairOutcome, "OpenClawGateway")); }, + _ => _resetBudgetCount++, + () => _enabled, + _diagnostics, + NullLogger.Instance, + _clock, + unhealthyThreshold: TimeSpan.FromSeconds(10), + cooldown: TimeSpan.FromSeconds(30), + startupGrace: startupGrace ?? TimeSpan.Zero, + delay: (_, _) => Task.CompletedTask, + isAutomaticRepairAllowed: isAutomaticRepairAllowed); + + private static GatewayConnectionSnapshot Op( + RoleConnectionState state, + string? error = null, + GatewayErrorKind? kind = null) => + new() { OperatorState = state, OperatorError = error, OperatorErrorKind = kind }; + + [Fact] + public async Task NotEligibleGateway_NeverRepairs() + { + SetActive(new GatewayRecord { Id = "gw-remote", Url = "wss://remote.example" }); + _snapshot = Op(RoleConnectionState.Error, "connection refused"); + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + _clock.Advance(TimeSpan.FromMinutes(5)); + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(0, _repairCount); + } + + [Fact] + public async Task KillSwitchDisabled_NeverRepairs() + { + SetActiveManagedLocal(); + _enabled = false; + _snapshot = Op(RoleConnectionState.Error, "connection refused"); + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + _clock.Advance(TimeSpan.FromMinutes(5)); + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(0, _repairCount); + } + + [Fact] + public async Task OperatorConnected_NoRepair_ResetsBudget() + { + SetActiveManagedLocal(); + _snapshot = Op(RoleConnectionState.Connected); + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(0, _repairCount); + Assert.True(_resetBudgetCount >= 1); + } + + [Fact] + public async Task SnapshotForDifferentGateway_IsIgnored_NoBudgetResetNoRepair() + { + SetActiveManagedLocal(); // active = gw-local + // During a switch the snapshot can still describe a DIFFERENT gateway. A Connected snapshot for + // gw-other must NOT reset gw-local's restart budget (which would let alternating gateways bypass + // the bound), and the tick must not act on gw-local at all. + _snapshot = new GatewayConnectionSnapshot + { + OperatorState = RoleConnectionState.Connected, + GatewayId = "gw-other", + }; + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(0, _resetBudgetCount); // gw-other's health not attributed to gw-local + Assert.Equal(0, _repairCount); + } + + [Fact] + public async Task OperatorIdle_DoesNotRepair_SoManualDisconnectStands() + { + SetActiveManagedLocal(); + _snapshot = Op(RoleConnectionState.Idle); + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + _clock.Advance(TimeSpan.FromMinutes(5)); + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(0, _repairCount); + } + + [Theory] + [InlineData("unauthorized")] + [InlineData("AUTH_DEVICE_TOKEN_MISMATCH")] + [InlineData("device token mismatch")] + [InlineData("rate limit exceeded")] + [InlineData("insufficient scope operator.admin")] + [InlineData("TLS handshake failed")] + public async Task AuthOrNonTransportError_DoesNotRepair(string operatorError) + { + SetActiveManagedLocal(); + _snapshot = Op(RoleConnectionState.Error, operatorError); + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + _clock.Advance(TimeSpan.FromMinutes(5)); + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(0, _repairCount); + } + + [Fact] + public async Task TypedTlsFailure_WithGenericTransportText_DoesNotRepair() + { + SetActiveManagedLocal(); + _snapshot = Op(RoleConnectionState.Error, "Transport error", GatewayErrorKind.Tls); + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + _clock.Advance(TimeSpan.FromMinutes(5)); + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(0, _repairCount); + } + + [Fact] + public async Task TypedLocalPortConflict_PastThreshold_Repairs() + { + SetActiveManagedLocal(); + _snapshot = Op( + RoleConnectionState.Error, + "gateway token mismatch", + GatewayErrorKind.LocalPortConflict); + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + _clock.Advance(TimeSpan.FromSeconds(11)); + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(1, _repairCount); + } + + [Fact] + public async Task UserDisconnectedIntent_NeverRepairs() + { + SetActiveManagedLocal(); + _snapshot = Op(RoleConnectionState.Error, "connection refused", GatewayErrorKind.Network); + await using var monitor = CreateMonitor( + isAutomaticRepairAllowed: _ => false); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + _clock.Advance(TimeSpan.FromMinutes(5)); + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(0, _repairCount); + } + + [Fact] + public async Task PairingRequired_DoesNotRepair() + { + SetActiveManagedLocal(); + _snapshot = new GatewayConnectionSnapshot + { + OperatorState = RoleConnectionState.Error, + OperatorPairingRequired = true + }; + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + _clock.Advance(TimeSpan.FromMinutes(5)); + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(0, _repairCount); + } + + [Fact] + public async Task TransportUnreachable_PastThreshold_Repairs() + { + SetActiveManagedLocal(); + _snapshot = Op(RoleConnectionState.Error, "connection refused"); + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); // starts unhealthy timer + _clock.Advance(TimeSpan.FromSeconds(11)); // past 10s threshold + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(1, _repairCount); + } + + [Fact] + public async Task TransportUnreachable_BelowThreshold_NoRepair() + { + SetActiveManagedLocal(); + _snapshot = Op(RoleConnectionState.Error, "connection refused"); + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + _clock.Advance(TimeSpan.FromSeconds(9)); // under threshold + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(0, _repairCount); + } + + [Fact] + public async Task WithinStartupGrace_DoesNotRepair_EvenWhenUnhealthy() + { + SetActiveManagedLocal(); + _snapshot = Op(RoleConnectionState.Connecting); // cold start: Connecting, no classified error + await using var monitor = CreateMonitor(startupGrace: TimeSpan.FromSeconds(45)); + + // Unhealthy well past the threshold, but still inside the startup grace window. + await monitor.EvaluateOnceAsync(CancellationToken.None); + _clock.Advance(TimeSpan.FromSeconds(20)); + await monitor.EvaluateOnceAsync(CancellationToken.None); + Assert.Equal(0, _repairCount); + + // After the grace window elapses and the threshold accrues, repair fires. + _clock.Advance(TimeSpan.FromSeconds(40)); // now past 45s grace + await monitor.EvaluateOnceAsync(CancellationToken.None); // re-arms timer post-grace + _clock.Advance(TimeSpan.FromSeconds(11)); + await monitor.EvaluateOnceAsync(CancellationToken.None); + Assert.Equal(1, _repairCount); + } + + [Fact] + public async Task WithinCooldown_DoesNotRepairAgain() + { + SetActiveManagedLocal(); + _snapshot = Op(RoleConnectionState.Error, "connection refused"); + _repairOutcome = ManagedLocalGatewayRepairOutcome.ReconnectPendingVerification; + await using var monitor = CreateMonitor(); + + await monitor.EvaluateOnceAsync(CancellationToken.None); + _clock.Advance(TimeSpan.FromSeconds(11)); + await monitor.EvaluateOnceAsync(CancellationToken.None); // repair #1 + _clock.Advance(TimeSpan.FromSeconds(20)); // within 30s cooldown + await monitor.EvaluateOnceAsync(CancellationToken.None); + + Assert.Equal(1, _repairCount); + } + + private sealed class FakeClock : IClock + { + public DateTime UtcNow { get; private set; } = new(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc); + public void Advance(TimeSpan by) => UtcNow += by; + } +} diff --git a/tests/OpenClaw.Tray.Tests/ManagedLocalGatewayRepairCoordinatorTests.cs b/tests/OpenClaw.Tray.Tests/ManagedLocalGatewayRepairCoordinatorTests.cs new file mode 100644 index 000000000..0bdd72b12 --- /dev/null +++ b/tests/OpenClaw.Tray.Tests/ManagedLocalGatewayRepairCoordinatorTests.cs @@ -0,0 +1,539 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using OpenClaw.Connection; +using OpenClaw.Shared; +using OpenClawTray.Services; +using Xunit; + +namespace OpenClaw.Tray.Tests; + +public class ManagedLocalGatewayRepairCoordinatorTests : IDisposable +{ + private readonly string _tempDir; + private readonly GatewayRegistry _registry; + private readonly ConnectionDiagnostics _diagnostics = new(); + + public ManagedLocalGatewayRepairCoordinatorTests() + { + _tempDir = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "openclaw-repair-" + Guid.NewGuid().ToString("N")[..8]); + System.IO.Directory.CreateDirectory(_tempDir); + _registry = new GatewayRegistry(_tempDir); + } + + public void Dispose() + { + // slopwatch-ignore: SW003 Test cleanup is best-effort and must not hide the test outcome. + try { System.IO.Directory.Delete(_tempDir, true); } catch { } + } + + private void SetActiveManagedLocal() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway" + }); + _registry.SetActive("gw-local"); + } + + private void AddManagedLocal(string id, string distro, int port) + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = id, + Url = $"ws://localhost:{port}", + IsLocal = true, + SetupManagedDistroName = distro + }); + } + + private ManagedLocalGatewayRepairCoordinator CreateCoordinator( + IManagedLocalGatewayRestarter restarter, + Func probeReachable, + Func> reconnectIfCurrent, + Func isOperatorConnected, + Action? onReArm = null, + int maxRestarts = 3, + Func? tryAcquireLifecycleLease = null, + Func? isRestartStillWarranted = null, + Func? isAutomaticRepairAllowed = null, + Func>? + repairPortConflictAsync = null, + Func? isPortConflictCandidate = null) + => new( + _registry, + restarter, + (_, _) => Task.FromResult(probeReachable()), + reconnectIfCurrent, + isOperatorConnected, + _ => { onReArm?.Invoke(); return Task.CompletedTask; }, + _diagnostics, + NullLogger.Instance, + maxRestartsPerGateway: maxRestarts, + verifyTimeout: TimeSpan.FromMilliseconds(150), + verifyPollInterval: TimeSpan.FromMilliseconds(10), + tryAcquireLifecycleLease: tryAcquireLifecycleLease, + isRestartStillWarranted: isRestartStillWarranted, + isAutomaticRepairAllowed: isAutomaticRepairAllowed, + repairPortConflictAsync: repairPortConflictAsync, + isPortConflictCandidate: isPortConflictCandidate); + + [Fact] + public async Task RemoteGateway_NotEligible_DoesNotRestart() + { + _registry.AddOrUpdate(new GatewayRecord { Id = "gw-remote", Url = "wss://remote.example" }); + _registry.SetActive("gw-remote"); + var restarter = new FakeRestarter(); + var coordinator = CreateCoordinator(restarter, () => false, (_, _) => Task.FromResult(true), () => true); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.NotEligible, result.Outcome); + Assert.Equal(0, restarter.Calls); + } + + [Fact] + public async Task SshTunnelGateway_NotEligible_DoesNotRestart() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-ssh", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + SshTunnel = new SshTunnelConfig("user", "host.example", 18789, 45678) + }); + _registry.SetActive("gw-ssh"); + var restarter = new FakeRestarter(); + var coordinator = CreateCoordinator(restarter, () => false, (_, _) => Task.FromResult(true), () => true); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.NotEligible, result.Outcome); + Assert.Equal(0, restarter.Calls); + } + + [Fact] + public async Task ProbeReachable_ReconnectsWithoutRestart() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter(); + var reconnects = 0; + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => true, + (_, _) => { reconnects++; return Task.FromResult(true); }, + () => true); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.ReconnectedWithoutRestart, result.Outcome); + Assert.Equal(0, restarter.Calls); // reachable -> no distro restart + Assert.Equal(1, reconnects); + } + + [Fact] + public async Task ProbeReachableButNeverVerifies_EscalatesToBudgetedRestart() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter(); + var reconnects = 0; + // Wedged-gateway case: the gateway answers TCP (reachable=true) so the reconnect "succeeds", + // but the operator connection never verifies until a distro restart actually happens. The + // coordinator must not loop forever on the TCP-positive reachable path — it must escalate to + // exactly one budgeted restart, then verify and report Repaired. + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => true, + (_, _) => { reconnects++; return Task.FromResult(true); }, + isOperatorConnected: () => restarter.Calls > 0); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.Repaired, result.Outcome); + Assert.Equal(1, restarter.Calls); // escalated to a single budgeted restart + Assert.Equal(2, reconnects); // reachable-path reconnect, then post-restart reconnect + } + + [Fact] + public async Task ProbeReachable_ReconnectReturnsFalse_AbortsWithoutRestart() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter(); + // Reachable, but the reconnect no-ops (returns false) because the active gateway changed + // during the repair. The coordinator must NOT escalate to restarting a gateway we left. + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => true, + (_, _) => Task.FromResult(false), + isOperatorConnected: () => false); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.AbortedGatewayChanged, result.Outcome); + Assert.Equal(0, restarter.Calls); + } + + [Fact] + public async Task GatewaySwitchDuringVerify_AbortsBeforeConsumingBudgetOrRestarting() + { + SetActiveManagedLocal(); + _registry.AddOrUpdate(new GatewayRecord { Id = "gw-other", Url = "wss://remote.example" }); + var restarter = new FakeRestarter(); + // Reachable + pinned reconnect returns true, but simulates the user switching gateways during + // it. Verify then fails (active != pinned). The coordinator must re-check the active gateway and + // ABORT before consuming budget or restarting the gateway the user just left. + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => true, + reconnectIfCurrent: (_, _) => { _registry.SetActive("gw-other"); return Task.FromResult(true); }, + isOperatorConnected: () => true); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.AbortedGatewayChanged, result.Outcome); + Assert.Equal(0, restarter.Calls); + } + + [Fact] + public async Task RecordRepointedToRemoteDuringVerify_AbortsBeforeRestart() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter(); + var mutated = false; + // Reachable + reconnect "succeeds" but never verifies; during verify the user repoints the SAME + // record to a remote host (same id, no longer a managed WSL gateway). The pre-restart guard must + // re-validate the full record and abort — not restart a WSL distro for a now-remote gateway. + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => true, + reconnectIfCurrent: (_, _) => Task.FromResult(true), + isOperatorConnected: () => + { + if (!mutated) + { + mutated = true; + _registry.AddOrUpdate(new GatewayRecord { Id = "gw-local", Url = "wss://remote.example" }); + } + return false; + }); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.AbortedGatewayChanged, result.Outcome); + Assert.Equal(0, restarter.Calls); + } + + [Fact] + public async Task RepairDeferred_WhenManualOpHoldsLifecycleLease() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter(); + // A manual gateway lifecycle op holds the shared lease → the coordinator's non-blocking acquire + // returns null, so it must defer the destructive restart rather than run it concurrently. + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => false, + reconnectIfCurrent: (_, _) => Task.FromResult(true), + isOperatorConnected: () => false, + tryAcquireLifecycleLease: () => null); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.AbortedGatewayChanged, result.Outcome); + Assert.Equal(0, restarter.Calls); + } + + [Fact] + public async Task RepairAborts_WhenFailureIsNoLongerTransport() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter(); + // The operator failure became a non-transport terminal error (e.g. auth) during verify — a + // distro restart cannot fix it, so the coordinator must abort instead of burning the budget. + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => false, + reconnectIfCurrent: (_, _) => Task.FromResult(true), + isOperatorConnected: () => false, + isRestartStillWarranted: () => false); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.AbortedNonTransportFailure, result.Outcome); + Assert.Equal(0, restarter.Calls); + } + + [Fact] + public async Task UserDisconnectedIntent_AbortsBeforeProbeOrRestart() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter(); + var probes = 0; + var coordinator = new ManagedLocalGatewayRepairCoordinator( + _registry, + restarter, + (_, _) => { probes++; return Task.FromResult(false); }, + (_, _) => Task.FromResult(true), + () => false, + _ => Task.CompletedTask, + _diagnostics, + NullLogger.Instance, + verifyTimeout: TimeSpan.FromMilliseconds(100), + isAutomaticRepairAllowed: _ => false); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.AbortedUserIntent, result.Outcome); + Assert.Equal(0, probes); + Assert.Equal(0, restarter.Calls); + } + + [Fact] + public async Task UserStopsDuringRestart_PostRestartReconnectDoesNotRun() + { + SetActiveManagedLocal(); + var allowed = true; + var reconnects = 0; + var restarter = new FakeRestarter + { + Gate = () => + { + allowed = false; + return Task.CompletedTask; + } + }; + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => false, + reconnectIfCurrent: (_, _) => { reconnects++; return Task.FromResult(true); }, + isOperatorConnected: () => false, + isAutomaticRepairAllowed: _ => allowed); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.AbortedUserIntent, result.Outcome); + Assert.Equal(1, restarter.Calls); + Assert.Equal(0, reconnects); + } + + [Fact] + public async Task ProvenNativePortConflict_IsRemovedBeforeWslRestart() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter(); + var portRepairCalls = 0; + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => true, + reconnectIfCurrent: (_, _) => Task.FromResult(true), + isOperatorConnected: () => true, + repairPortConflictAsync: (_, _) => + { + portRepairCalls++; + return Task.FromResult(new ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome.Repaired)); + }, + isPortConflictCandidate: () => true); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.Repaired, result.Outcome); + Assert.Equal(1, portRepairCalls); + Assert.Equal(1, restarter.Calls); // skips reachable probe path and rebinds WSL relay + } + + [Fact] + public async Task UnknownPortOwner_IsBlockedWithoutRestart() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter(); + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => true, + reconnectIfCurrent: (_, _) => Task.FromResult(true), + isOperatorConnected: () => false, + repairPortConflictAsync: (_, _) => Task.FromResult( + new ManagedLocalPortConflictRepairResult( + ManagedLocalPortConflictRepairOutcome.BlockedUnknownOwner, + "unknown owner")), + isPortConflictCandidate: () => true); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.PortConflictBlocked, result.Outcome); + Assert.Equal(0, restarter.Calls); + } + + [Fact] + public async Task RestartBudget_IsPerGateway_AlternatingDoesNotResetOtherGateway() + { + // Two managed-local gateways: A exhausts its own restart budget; B has an independent budget; + // returning to A does NOT grant a fresh budget (the bound is keyed per gateway id, so + // alternating gateways cannot bypass it). + AddManagedLocal("gw-a", "DistroA", 18801); + AddManagedLocal("gw-b", "DistroB", 18802); + var restarter = new FakeRestarter(); + // Unreachable + never verifies: each attempt restarts once then ends PendingVerification. + var coordinator = CreateCoordinator( + restarter, () => false, (_, _) => Task.FromResult(true), () => false, maxRestarts: 1); + + _registry.SetActive("gw-a"); + Assert.Equal(ManagedLocalGatewayRepairOutcome.ReconnectPendingVerification, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); + Assert.Equal(ManagedLocalGatewayRepairOutcome.AttemptsExhausted, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); + + _registry.SetActive("gw-b"); + Assert.Equal(ManagedLocalGatewayRepairOutcome.ReconnectPendingVerification, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); + Assert.Equal(ManagedLocalGatewayRepairOutcome.AttemptsExhausted, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); + + _registry.SetActive("gw-a"); + Assert.Equal(ManagedLocalGatewayRepairOutcome.AttemptsExhausted, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); + + Assert.Equal(2, restarter.Calls); // exactly one restart for A and one for B; A never restarted twice + } + + [Fact] + public async Task ResetAttemptBudget_ForOneGateway_DoesNotResetAnother() + { + AddManagedLocal("gw-a", "DistroA", 18811); + AddManagedLocal("gw-b", "DistroB", 18812); + var restarter = new FakeRestarter(); + var coordinator = CreateCoordinator( + restarter, () => false, (_, _) => Task.FromResult(true), () => false, maxRestarts: 1); + + _registry.SetActive("gw-a"); + await coordinator.TryRepairActiveGatewayAsync(); // A restart #1 + Assert.Equal(ManagedLocalGatewayRepairOutcome.AttemptsExhausted, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); + + _registry.SetActive("gw-b"); + await coordinator.TryRepairActiveGatewayAsync(); // B restart #1 + Assert.Equal(ManagedLocalGatewayRepairOutcome.AttemptsExhausted, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); + + // Reset ONLY gateway A's budget. + coordinator.ResetAttemptBudget("gw-a"); + + _registry.SetActive("gw-a"); + Assert.Equal(ManagedLocalGatewayRepairOutcome.ReconnectPendingVerification, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); // A can restart again + _registry.SetActive("gw-b"); + Assert.Equal(ManagedLocalGatewayRepairOutcome.AttemptsExhausted, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); // B still exhausted + } + + [Fact] + public async Task ProbeUnreachable_RestartsRearmsReconnectsAndVerifies() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter(); + var reconnects = 0; + var rearms = 0; + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => false, + (_, _) => { reconnects++; return Task.FromResult(true); }, + () => true, + onReArm: () => rearms++); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.Repaired, result.Outcome); + Assert.Equal("OpenClawGateway", result.DistroName); + Assert.Equal(1, restarter.Calls); + Assert.Equal(1, rearms); // keepalive re-armed after restart + Assert.Equal(1, reconnects); + } + + [Fact] + public async Task RestartFails_DoesNotReconnect() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter { Result = new ManagedLocalGatewayRestartResult(false, "boom") }; + var reconnects = 0; + var coordinator = CreateCoordinator( + restarter, () => false, (_, _) => { reconnects++; return Task.FromResult(true); }, () => true); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.RestartFailed, result.Outcome); + Assert.Equal(1, restarter.Calls); + Assert.Equal(0, reconnects); + } + + [Fact] + public async Task RestartBudget_ExhaustsThenResetAllowsAgain() + { + SetActiveManagedLocal(); + var restarter = new FakeRestarter(); + // Unreachable + operator never connects: each attempt restarts and ends PendingVerification. + var coordinator = CreateCoordinator( + restarter, () => false, (_, _) => Task.FromResult(true), () => false, maxRestarts: 2); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.ReconnectPendingVerification, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); + Assert.Equal(ManagedLocalGatewayRepairOutcome.ReconnectPendingVerification, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); + Assert.Equal(ManagedLocalGatewayRepairOutcome.AttemptsExhausted, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); + Assert.Equal(2, restarter.Calls); // exhausted attempt did not restart again + + coordinator.ResetAttemptBudget(); + + Assert.Equal(ManagedLocalGatewayRepairOutcome.ReconnectPendingVerification, (await coordinator.TryRepairActiveGatewayAsync()).Outcome); + Assert.Equal(3, restarter.Calls); + } + + [Fact] + public async Task CancelledDuringRestart_DoesNotReconnect() + { + SetActiveManagedLocal(); + using var cts = new CancellationTokenSource(); + var reconnects = 0; + var restarter = new FakeRestarter { Gate = () => { cts.Cancel(); return Task.CompletedTask; } }; + var coordinator = CreateCoordinator( + restarter, () => false, (_, _) => { reconnects++; return Task.FromResult(true); }, () => true); + + await Assert.ThrowsAnyAsync( + () => coordinator.TryRepairActiveGatewayAsync(cts.Token)); + + Assert.Equal(1, restarter.Calls); + Assert.Equal(0, reconnects); + } + + [Fact] + public async Task GatewaySwitchDuringRepair_PinnedReconnectDoesNotReportRepaired() + { + SetActiveManagedLocal(); + _registry.AddOrUpdate(new GatewayRecord { Id = "gw-other", Url = "wss://remote.example" }); + var restarter = new FakeRestarter(); + // The reconnect simulates a gateway switch: pinned reconnect returns false (active changed). + var coordinator = CreateCoordinator( + restarter, + probeReachable: () => false, + reconnectIfCurrent: (_, _) => { _registry.SetActive("gw-other"); return Task.FromResult(false); }, + isOperatorConnected: () => true); + + var result = await coordinator.TryRepairActiveGatewayAsync(); + + // Restart targeted the ORIGINAL distro (captured before the switch), but the pinned reconnect + // no-ops once the active gateway changed — so the repair aborts cleanly rather than reporting a + // false success (or driving the switched-to gateway). + Assert.Equal("OpenClawGateway", restarter.LastDistro); + Assert.Equal(ManagedLocalGatewayRepairOutcome.AbortedGatewayChanged, result.Outcome); + } + + private sealed class FakeRestarter : IManagedLocalGatewayRestarter + { + public int Calls; + public string? LastDistro; + public ManagedLocalGatewayRestartResult Result = new(true); + public Func? Gate; + + public async Task RestartAsync(string distroName, CancellationToken cancellationToken) + { + Calls++; + LastDistro = distroName; + if (Gate != null) + await Gate(); + cancellationToken.ThrowIfCancellationRequested(); + return Result; + } + } +} diff --git a/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj b/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj index 50cfa608e..dd69a7bb5 100644 --- a/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj +++ b/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj @@ -72,6 +72,9 @@ + + + diff --git a/tests/OpenClaw.Tray.Tests/WslGatewayControllerTests.cs b/tests/OpenClaw.Tray.Tests/WslGatewayControllerTests.cs index 5358f14dd..9bc0909d4 100644 --- a/tests/OpenClaw.Tray.Tests/WslGatewayControllerTests.cs +++ b/tests/OpenClaw.Tray.Tests/WslGatewayControllerTests.cs @@ -74,12 +74,78 @@ public async Task RunAsync_AttemptsCommand_WhenDistroEnumerationIsEmpty() Assert.DoesNotContain("not registered", result.StandardError); } + [Fact] + public async Task ForceRestartAsync_TerminatesDistroThenColdRestarts() + { + var runner = new FakeWslCommandRunner + { + Distros = [new WslDistroInfo("OpenClawGateway", "Running", 2)], + Result = new WslCommandResult(0, "ok", string.Empty), + }; + var controller = new WslGatewayController(runner, NullLogger.Instance); + + var result = await controller.ForceRestartAsync("OpenClawGateway"); + + Assert.True(result.Success); + Assert.Equal(1, runner.TerminateCount); // host-side terminate happened first + Assert.Equal("OpenClawGateway", runner.LastTerminatedDistro); + Assert.Equal( // then a cold in-distro restart + WslGatewayControlCommandBuilder.Build(WslGatewayControlAction.Restart), + runner.LastDistroCommand); + } + + [Fact] + public async Task Restarter_WhenInPlaceRestartFails_EscalatesToTerminateAndForceRestart() + { + // Wedged-VM case: the in-place `gateway restart` fails; the restarter must escalate to a + // host-side terminate + cold restart rather than giving up (Sol-A). + var runner = new FakeWslCommandRunner + { + Distros = [new WslDistroInfo("OpenClawGateway", "Running", 2)], + InDistroResults = new Queue( + [ + new WslCommandResult(1, string.Empty, "wedged"), // in-place restart fails + new WslCommandResult(0, "ok", string.Empty), // cold restart after terminate succeeds + ]), + }; + var controller = new WslGatewayController(runner, NullLogger.Instance); + var restarter = new OpenClawTray.Services.WslManagedLocalGatewayRestarter(controller); + + var result = await restarter.RestartAsync("OpenClawGateway", CancellationToken.None); + + Assert.True(result.Success); + Assert.Equal(1, runner.TerminateCount); // escalated: host-side terminate happened + Assert.Equal(2, runner.InDistroCount); // in-place attempt + post-terminate cold restart + } + + [Fact] + public async Task Restarter_WhenBothRestartsFail_ReportsFailure() + { + var runner = new FakeWslCommandRunner + { + Distros = [new WslDistroInfo("OpenClawGateway", "Running", 2)], + Result = new WslCommandResult(1, string.Empty, "still wedged"), + }; + var controller = new WslGatewayController(runner, NullLogger.Instance); + var restarter = new OpenClawTray.Services.WslManagedLocalGatewayRestarter(controller); + + var result = await restarter.RestartAsync("OpenClawGateway", CancellationToken.None); + + Assert.False(result.Success); + Assert.Equal(1, runner.TerminateCount); // escalation was attempted + Assert.Equal(2, runner.InDistroCount); + } + private sealed class FakeWslCommandRunner : IWslCommandRunner { public IReadOnlyList Distros { get; init; } = []; public WslCommandResult Result { get; init; } = new(0, string.Empty, string.Empty); + public Queue? InDistroResults { get; init; } public string? LastDistroName { get; private set; } public IReadOnlyList? LastDistroCommand { get; private set; } + public int InDistroCount { get; private set; } + public int TerminateCount { get; private set; } + public string? LastTerminatedDistro { get; private set; } public Task RunAsync( IReadOnlyList arguments, @@ -96,7 +162,9 @@ public Task> ListDistrosAsync(CancellationToken can public Task TerminateDistroAsync(string name, CancellationToken cancellationToken = default) { - return Task.FromResult(Result); + TerminateCount++; + LastTerminatedDistro = name; + return Task.FromResult(new WslCommandResult(0, string.Empty, string.Empty)); } public Task UnregisterDistroAsync(string name, CancellationToken cancellationToken = default) @@ -112,7 +180,9 @@ public Task RunInDistroAsync( { LastDistroName = name; LastDistroCommand = command; - return Task.FromResult(Result); + InDistroCount++; + var result = InDistroResults is { Count: > 0 } ? InDistroResults.Dequeue() : Result; + return Task.FromResult(result); } } } diff --git a/tests/OpenClaw.Tray.Tests/WslKeepAlivePolicyTests.cs b/tests/OpenClaw.Tray.Tests/WslKeepAlivePolicyTests.cs index 1547bd479..88aebb711 100644 --- a/tests/OpenClaw.Tray.Tests/WslKeepAlivePolicyTests.cs +++ b/tests/OpenClaw.Tray.Tests/WslKeepAlivePolicyTests.cs @@ -5,6 +5,25 @@ namespace OpenClaw.Tray.Tests; public class WslKeepAlivePolicyTests { + [Fact] + public void MarkedKeepaliveIdentity_RejectsReusedPidProcessNameOrStartTime() + { + var markerStart = new DateTime(2026, 7, 24, 1, 2, 3, DateTimeKind.Utc); + + Assert.True(WslKeepAlivePolicy.IsMarkedKeepaliveProcessIdentity( + "wsl", + markerStart.AddSeconds(1), + markerStart)); + Assert.False(WslKeepAlivePolicy.IsMarkedKeepaliveProcessIdentity( + "svchost", + markerStart, + markerStart)); + Assert.False(WslKeepAlivePolicy.IsMarkedKeepaliveProcessIdentity( + "wsl", + markerStart.AddMinutes(1), + markerStart)); + } + [Fact] public void ShouldStart_UsesActiveLocalRegistryRecord_WhenLegacySettingsAreEmpty() {