From 6ca7d43f7423dafed690c32b127f2b6684b927a8 Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 20:22:50 -0700 Subject: [PATCH 01/12] feat: scope sandbox repository credentials Mint run-bound repository credentials from immutable snapshots and provide them only to approved sandbox git or gh commands. Revoke credentials on release and orphan cleanup. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- .../AgentHostRuntimeState.cs | 18 +- apps/Agentweaver.AgentHost/Program.cs | 11 +- .../RunScopedRepositoryCredentialProvider.cs | 10 + .../Auth/GitHubCapabilityBroker.cs | 45 + apps/Agentweaver.Api/Program.cs | 1 + .../Sandbox/AgentHostReaperService.cs | 21 +- .../Sandbox/KubernetesSandboxExecutor.cs | 3049 +++++++++-------- .../RunRepositoryCredentialRegistry.cs | 93 + .../Sandbox/SandboxExecutorRouter.cs | 365 +- .../Webhooks/RepoAppInstallationService.cs | 1120 +++--- docs/deep-dive/sandboxed-execution.md | 6 + .../CopilotAIAgent.cs | 6 +- .../ISandboxRepositoryCredentialProvider.cs | 7 + .../SandboxToolOptions.cs | 6 + .../Tools/RunCommandTool.cs | 54 +- packages/Agentweaver.Domain/SandboxPolicy.cs | 6 +- .../Auth/TwoAppCredentialArchitectureTests.cs | 31 +- .../AssemblyBuildTestShellGuardTests.cs | 105 +- 18 files changed, 2697 insertions(+), 2257 deletions(-) create mode 100644 apps/Agentweaver.AgentHost/RunScopedRepositoryCredentialProvider.cs create mode 100644 apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs create mode 100644 packages/Agentweaver.AgentTools/ISandboxRepositoryCredentialProvider.cs diff --git a/apps/Agentweaver.AgentHost/AgentHostRuntimeState.cs b/apps/Agentweaver.AgentHost/AgentHostRuntimeState.cs index 85703eedf..2d28ff2b5 100644 --- a/apps/Agentweaver.AgentHost/AgentHostRuntimeState.cs +++ b/apps/Agentweaver.AgentHost/AgentHostRuntimeState.cs @@ -87,6 +87,12 @@ internal sealed class AgentHostRuntimeState /// public string? GitHubAccessToken { get; private set; } + /// + /// Short-lived installation credential for the configured run and repository. The shell tool + /// passes this value only to a simple git or gh child process. + /// + public string? RepositoryAccessToken { get; private set; } + /// /// The authenticated platform caller token forwarded only for operator-assistant MCP requests. /// This is distinct from : in Entra deployments the former is the @@ -107,6 +113,7 @@ public void InitializeFromOptions(AgentHostOptions options) PreviewRunnerCredential = string.Empty; // not available on env-var launch path KvUserSecretName = options.KvUserSecretName; GitHubAccessToken = null; // not available on env-var launch path + RepositoryAccessToken = null; CallerBearerToken = null; // operator-assistant-only warm-pod input Purpose = AgentHostPurpose.Default; WorkspaceMode = ExecutionWorkspaceMode.Shared; @@ -127,7 +134,7 @@ public void InitializeFromOptions(AgentHostOptions options) /// Atomically transitions the pod from standby to configured. Returns /// when the pod was already configured (one-time semantics → caller returns 409). /// - public bool TryConfigure(string runId, string userId, string turnBearerToken, string? kvUserSecretName, string? gitHubAccessToken, string? previewRunnerCredential = null) + public bool TryConfigure(string runId, string userId, string turnBearerToken, string? kvUserSecretName, string? gitHubAccessToken, string? previewRunnerCredential = null, string? repositoryAccessToken = null) => TryConfigure(new AgentHostRunConfiguration( runId, userId, @@ -135,7 +142,8 @@ public bool TryConfigure(string runId, string userId, string turnBearerToken, st kvUserSecretName, gitHubAccessToken, previewRunnerCredential, - SharedWorkingDirectory: null)); + SharedWorkingDirectory: null, + RepositoryAccessToken: repositoryAccessToken)); /// Atomically applies the complete run-scoped warm-pod configuration. public bool TryConfigure(AgentHostRunConfiguration configuration) @@ -153,6 +161,9 @@ public bool TryConfigure(AgentHostRunConfiguration configuration) GitHubAccessToken = string.IsNullOrWhiteSpace(configuration.GitHubAccessToken) ? null : configuration.GitHubAccessToken; + RepositoryAccessToken = string.IsNullOrWhiteSpace(configuration.RepositoryAccessToken) + ? null + : configuration.RepositoryAccessToken; CallerBearerToken = string.IsNullOrWhiteSpace(configuration.CallerBearerToken) ? null : configuration.CallerBearerToken; @@ -208,4 +219,5 @@ internal sealed record AgentHostRunConfiguration( string? CommitAuthorEmail = null, string? ProjectId = null, string? AgentName = null, - string? CallerBearerToken = null); + string? CallerBearerToken = null, + string? RepositoryAccessToken = null); diff --git a/apps/Agentweaver.AgentHost/Program.cs b/apps/Agentweaver.AgentHost/Program.cs index b67c22a85..503af706b 100644 --- a/apps/Agentweaver.AgentHost/Program.cs +++ b/apps/Agentweaver.AgentHost/Program.cs @@ -2,6 +2,7 @@ using Agentweaver.AgentHost; using Agentweaver.AgentRuntime; using Agentweaver.AgentRuntime.Providers; +using Agentweaver.AgentTools; using Agentweaver.Domain; using Agentweaver.SandboxExec; using Agentweaver.SandboxExec.PodExec; @@ -130,6 +131,7 @@ // ── Sandbox policy (no DB in pod) ───────────────────────────────────────────── builder.Services.AddSingleton(); +builder.Services.AddSingleton(); // ── Agent runtime (in-memory approvals, local executor — Kata VM IS the sandbox) ─ builder.Services.AddSingleton(); @@ -588,6 +590,12 @@ internal sealed record ConfigureRequest /// public string? GitHubAccessToken { get; init; } + /// + /// Short-lived credential for the configured run and repository. The runtime gives this value + /// only to a single git or gh shell command. + /// + public string? RepositoryAccessToken { get; init; } + /// /// Authenticated platform caller token used by the operator assistant's MCP connection. Kept /// separate from because Entra deployments use different @@ -679,7 +687,8 @@ internal sealed record ConfigureRequest CommitAuthorEmail, ProjectId, AgentName, - CallerBearerToken); + CallerBearerToken, + RepositoryAccessToken); } internal sealed record PreviewProcessStartRequest diff --git a/apps/Agentweaver.AgentHost/RunScopedRepositoryCredentialProvider.cs b/apps/Agentweaver.AgentHost/RunScopedRepositoryCredentialProvider.cs new file mode 100644 index 000000000..ce03a59ec --- /dev/null +++ b/apps/Agentweaver.AgentHost/RunScopedRepositoryCredentialProvider.cs @@ -0,0 +1,10 @@ +using Agentweaver.AgentTools; + +namespace Agentweaver.AgentHost; + +/// Provides the in-memory repository credential for the configured run. +internal sealed class RunScopedRepositoryCredentialProvider( + AgentHostRuntimeState runtimeState) : ISandboxRepositoryCredentialProvider +{ + public string? GetAccessToken() => runtimeState.RepositoryAccessToken; +} diff --git a/apps/Agentweaver.Api/Auth/GitHubCapabilityBroker.cs b/apps/Agentweaver.Api/Auth/GitHubCapabilityBroker.cs index 822374a65..9f2938601 100644 --- a/apps/Agentweaver.Api/Auth/GitHubCapabilityBroker.cs +++ b/apps/Agentweaver.Api/Auth/GitHubCapabilityBroker.cs @@ -85,6 +85,51 @@ internal sealed class GitHubCapabilityBroker( : (GitHubCapabilityBrokerOutcome.Issued, new(fenced.Purpose, operation, expiresAt)); } + /// + /// Mints one short-lived installation credential after snapshot fencing. The caller receives + /// the value only in its callback. The API never uses this value for a GitHub command. + /// + internal async Task TryUseRepositoryCredentialAsync( + SnapshotRef snapshotRef, + DateTimeOffset now, + Func useCredential, + CancellationToken ct) + { + var fenced = await persistence.TryFenceLiveSnapshotAsync( + GitHubCapabilityPurpose.UnattendedRepository, + snapshotRef, + now, + ct).ConfigureAwait(false); + if (fenced is null) + return GitHubCapabilityBrokerOutcome.CapabilityUnavailable; + + string? token = null; + DateTimeOffset? expiresAt = null; + var outcome = await installationTokens.MintForRepositoryAsync( + fenced.InstallationId!.Value, + fenced.RepositoryId!.Value, + (value, expires) => + { + token = value; + expiresAt = expires; + return Task.CompletedTask; + }, + ct).ConfigureAwait(false); + if (outcome != RepoAppInstallationOutcome.Success || string.IsNullOrWhiteSpace(token) || + expiresAt is null || expiresAt <= now) + return GitHubCapabilityBrokerOutcome.CapabilityUnavailable; + + if (await persistence.TryFenceLiveSnapshotAsync( + GitHubCapabilityPurpose.UnattendedRepository, + snapshotRef, + now, + ct).ConfigureAwait(false) is null) + return GitHubCapabilityBrokerOutcome.CapabilityUnavailable; + + await useCredential(token, expiresAt.Value).ConfigureAwait(false); + return GitHubCapabilityBrokerOutcome.Issued; + } + internal static bool IsOperationAllowed( GitHubCapabilityPurpose purpose, GitHubCapabilityOperation operation) => diff --git a/apps/Agentweaver.Api/Program.cs b/apps/Agentweaver.Api/Program.cs index e855edb5d..6fec6512d 100644 --- a/apps/Agentweaver.Api/Program.cs +++ b/apps/Agentweaver.Api/Program.cs @@ -486,6 +486,7 @@ // AgentHost__UserId, scoping the in-pod GitHub Copilot auth to the user's Copilot-entitled token // instead of the installation token (which fails the first model turn). builder.Services.AddSingleton(); +builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => new RunAgentHostContextResolver( sp.GetRequiredService(), diff --git a/apps/Agentweaver.Api/Sandbox/AgentHostReaperService.cs b/apps/Agentweaver.Api/Sandbox/AgentHostReaperService.cs index c84e9cee7..b8d8440c0 100644 --- a/apps/Agentweaver.Api/Sandbox/AgentHostReaperService.cs +++ b/apps/Agentweaver.Api/Sandbox/AgentHostReaperService.cs @@ -41,6 +41,7 @@ public sealed class AgentHostReaperService : IAgentHostReaper // First-class preview lifecycle reconciler. Before reaping an orphaned claim, it derives durable // Previewable/PreviewActive state and atomically owns all retention or cleanup side effects. private readonly Preview.ISandboxPreviewService? _previewService; + private readonly RunRepositoryCredentialRegistry? _repositoryCredentials; public AgentHostReaperService( IKubernetes client, @@ -48,7 +49,8 @@ public AgentHostReaperService( KubernetesSandboxOptions options, ILogger logger, ISecretStore? secretStore = null, - Preview.ISandboxPreviewService? previewService = null) + Preview.ISandboxPreviewService? previewService = null, + RunRepositoryCredentialRegistry? repositoryCredentials = null) { _client = client; _runStore = runStore; @@ -56,6 +58,7 @@ public AgentHostReaperService( _logger = logger; _secretStore = secretStore; _previewService = previewService; + _repositoryCredentials = repositoryCredentials; } /// @@ -105,6 +108,7 @@ public async Task SweepOrphanedPodsAsync(CancellationToken ct = default) // from the claim annotation and delete it so the credential's durable lifetime stays // bounded by the pod's (spec-006 decouple-preview; no-op when absent). await TryDeleteOrphanCredentialAsync(claim.AnnotatedRunId, ct).ConfigureAwait(false); + await TryRevokeOrphanRepositoryCredentialAsync(claim.AnnotatedRunId, ct).ConfigureAwait(false); } } @@ -251,6 +255,21 @@ await _secretStore.DeleteSecretAsync(Preview.PreviewRunnerCredential.SecretKey(r } } + private async Task TryRevokeOrphanRepositoryCredentialAsync(string? runId, CancellationToken ct) + { + if (_repositoryCredentials is null || string.IsNullOrWhiteSpace(runId)) + return; + + try + { + await _repositoryCredentials.RevokeAsync(runId, ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, "AgentHostReaper: failed to revoke repository credential for run {RunId}", runId); + } + } + /// /// Reconciles every preview-retention side effect before deciding whether to reap. A missing /// service/run id or reconciliation failure defaults to Previewable (leak-safe) rather than diff --git a/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs b/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs index 48e092c00..4c2425901 100644 --- a/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs +++ b/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs @@ -1,1510 +1,1539 @@ -using System.Runtime.CompilerServices; -using System.Security.Cryptography; -using System.Text; -using System.Net.Http.Json; -using System.Net.Sockets; -using System.Text.Json; -using Agentweaver.Api.Auth; -using Agentweaver.Api.Infrastructure; -using Agentweaver.AgentRuntime.Workflow; -using Agentweaver.Domain; -using k8s; -using k8s.Autorest; -using Agentweaver.SandboxExec; -using Microsoft.Extensions.Logging; - -namespace Agentweaver.Api.Sandbox; - -/// -/// Configures the Kubernetes SandboxClaim backend. -/// Bound from the Sandbox:Kubernetes configuration section. -/// -public sealed class KubernetesSandboxOptions -{ - public string Namespace { get; init; } = "agentweaver"; - public string TemplateRef { get; init; } = "agentweaver-sandbox"; - /// - /// SandboxWarmPool the generic command-exec claim binds to. In the v1beta1 CRD a - /// SandboxClaim references a SandboxWarmPool (spec.warmPoolRef.name), - /// which in turn references the SandboxTemplate. Default: agentweaver-sandbox. - /// - public string WarmPoolRef { get; init; } = "agentweaver-sandbox"; - /// Path where the shared workspace PVC is mounted inside API and sandbox pods. - public string WorkspaceMountPath { get; init; } = "/workspace"; - /// SandboxClaim TTL. Command timeouts are capped below this so controller GC cannot interrupt exec. - public int TimeoutSeconds { get; init; } = 600; - /// Cluster service CIDR that must be excluded by sandbox egress policy. - public string? ServiceCidr { get; init; } - public IReadOnlyList SandboxEgressCidrExclusions { get; init; } = []; - - // ── Pod-per-run AgentHost lifecycle options (spec §9 / Q3 hybrid) ───────── - - /// - /// SandboxWarmPool the AgentHost (pod-per-run) claim binds to in the v0.5.0 v1beta1 CRD - /// (spec.warmPoolRef.name). The pool itself references the AgentHost SandboxTemplate. - /// Default: agentweaver-agent-host. - /// - public string AgentHostWarmPoolRef { get; init; } = "agentweaver-agent-host"; - - /// - /// Port the AgentHost Kestrel listener binds to inside the pod. - /// Worker builds the A2A endpoint as http://<podIP>:<AgentHostPort><AgentHostA2APath>. - /// TLS/mTLS termination is owned by Link (H1) — leave hook here for cert wiring. - /// Default: 8088. - /// - public int AgentHostPort { get; init; } = 8088; - - /// - /// A2A path prefix mounted by MapA2AHttpJson inside the AgentHost pod. - /// Must match AgentHost:A2APath set in the pod's configuration. - /// Default: /a2a/agent. - /// - public string AgentHostA2APath { get; init; } = "/a2a/agent"; - - /// - /// When (default) the AgentHost A2A endpoint uses https with - /// mTLS (H1). When (PoC only) it uses plain http. Drives the - /// scheme via and is injected into the pod as - /// AgentHost__RequireMtls. Config key: Sandbox:AgentHost:RequireMtls. - /// - public bool RequireMtls { get; init; } = true; - - // ── AgentHost readiness gate (A2A cold-start race) ─────────────────────── - - /// - /// Path the AgentHost exposes for liveness/readiness on . The executor - /// polls {scheme}://{podIP}:{port}{AgentHostHealthzPath} after the claim binds and BEFORE - /// returning the A2A endpoint, so the worker never sends the first turn into the Kestrel boot - /// window (which would be refused). Default: /healthz. - /// - public string AgentHostHealthzPath { get; init; } = "/healthz"; - - /// - /// Maximum time to wait for the AgentHost to start serving - /// before failing the launch deterministically. Default: 90s (covers cold-start Kestrel bind). - /// - public int AgentHostReadyTimeoutSeconds { get; init; } = 90; - - /// Interval between AgentHost readiness probe attempts. Default: 1000ms. - public int AgentHostReadyPollIntervalMs { get; init; } = 1000; - - /// - /// Minimum age before the orphan reaper may delete an AgentHost claim that is absent from the - /// active-run map. Config key: Sandbox:Kubernetes:AgentHostClaimCreationGraceSeconds. - /// The effective value is floored above . - /// Default: 300s. - /// - public int AgentHostClaimCreationGraceSeconds { get; init; } = 300; - - /// - /// Azure Key Vault URI injected into AgentHost pods as AgentHost__KeyVaultUri so the - /// warm pod can fetch the run owner's GitHub token via workload identity at /configure-time - /// (Option C). Sourced from the API's own KV config (Auth:TokenStore:KeyVaultUri). When - /// null/empty the env var is omitted and the pod falls back to the CSI file-mount path. - /// - public string? KvUri { get; init; } -} - -/// -/// Top-level sandbox runtime options bound from the Sandbox configuration section -/// (not under Sandbox:Kubernetes). Controls the agent-execution mode and -/// the pod-release-on-suspend behaviour (Q3 hybrid). -/// -public sealed class SandboxRuntimeOptions -{ - /// - /// Agent execution mode. - /// - /// in-api (default) — run agents in-process; instant rollback path (§4.7.6). - /// pod-per-run — launch a per-run AgentHost sandbox pod; activate A2A transport. - /// - /// - public string AgentExecutionMode { get; init; } = "in-api"; - - /// - /// When true (default) and is pod-per-run, - /// the AgentHost pod is released (SandboxClaim deleted) whenever the MAF graph suspends - /// at a RequestPort (HITL/review gate) or the coordinator idles awaiting children. - /// Set to false to keep the pod warm across suspension (lower resume latency, higher - /// resource cost; recommended only for short-wait HITL in dev/staging). - /// - public bool ReleasePodOnSuspend { get; init; } = true; - - /// - public bool IsPodPerRun => - string.Equals(AgentExecutionMode, "pod-per-run", StringComparison.OrdinalIgnoreCase); -} - -/// -/// Executes sandboxed commands inside a pre-warmed Kubernetes pod obtained via a -/// SandboxClaim CRD. Lifecycle: -/// -/// Create a SandboxClaim resource (adopts a warm pod from the pool). -/// Poll until the claim transitions to phase: Bound and reports a pod name. -/// Run the command via pod-exec (Kubernetes WebSocket exec API). -/// Delete the claim on completion (controller GC cleans up the pod and service). -/// -/// Automatically selected by the API when KUBERNETES_SERVICE_HOST is present -/// (see ). -/// -internal sealed class KubernetesSandboxExecutor : ISandboxExecutor, IAgentHostPodLifecycle -{ - private const string ApiGroup = SandboxClaimConventions.ApiGroup; - private const string ApiVersion = SandboxClaimConventions.ApiVersion; - private const string ClaimPlural = SandboxClaimConventions.ClaimPlural; - private const string ContainerName = "agentweaver-sandbox"; - - /// - /// Bounded attempt count for — the total number of - /// tries (initial + retries) for a transient Kubernetes API fault (issue #230). A transient - /// connection reset (SocketException 104 → IOException → HttpRequestException) that used to fail - /// a subtask fatally is now retried with exponential backoff + jitter. - /// - private const int MaxK8sAttempts = 3; - - /// - /// Cadence for the heartbeat emitted while an - /// AgentHost SandboxClaim is still being provisioned (unbound). Must stay well under the - /// parent coordinator's Coordinator:SubtaskStallTimeoutMinutes (default 5 min) so each - /// provisioning wait window is punctuated by an event that keeps the outbound stream flowing and - /// resets the stall timer (issue #217, mirrors the #212 tool.approval_pending heartbeat cadence). - /// - internal static readonly TimeSpan SandboxProvisioningHeartbeatInterval = TimeSpan.FromSeconds(20); - - private readonly IKubernetes _client; - private readonly KubernetesSandboxOptions _options; - private readonly ILogger _logger; - private readonly IPodNameRegistry? _podRegistry; - private readonly IAgentHostTurnTokenRegistry? _turnTokenRegistry; - private readonly Security.IRunAuthorshipCapabilityStore? _authorshipCapabilityStore; - // Polls the AgentHost /healthz after bind and before returning the endpoint, closing the - // A2A cold-start race (pod Running ~20-30s before Kestrel binds :8088). Null in unit tests - // that only assert the claim body → readiness gate is skipped. - private readonly IAgentHostReadinessProbe? _readinessProbe; - // Resolves the run's submitting user so the pod can be scoped (via /configure) to the run owner's - // Copilot-entitled token instead of the installation token. Null when the run→user lookup is - // unavailable. - private readonly IRunSubmittingUserResolver? _submittingUserResolver; - // Used to POST /configure to the warm pod after bind (warm-pool deferred-config path). Null in - // unit tests → the /configure call is skipped (same null-skip convention as the readiness probe). - private readonly IHttpClientFactory? _httpClientFactory; - // Resolves the run owner's GitHub token so the API can pass it in /configure, avoiding the need - // for the kata VM pod to call Azure AD or Key Vault (blocked by Cilium FQDN policies). - private readonly IGitHubTokenStore? _tokenStore; - private readonly IGitHubTokenScopeProvider? _tokenScopeProvider; - // Refresh-aware token accessor (issue #523): a Build & Test gate can launch its AgentHost pod for - // the FIRST time (a fresh pod, not yet /configure'd for this run) many minutes after the run's - // earlier subtask stages — long enough for the submitting user's Copilot-entitled OAuth access - // token to cross its expiry skew window. Reading the raw entry via IGitHubTokenStore.GetAsync (as - // ResolveGitHubAccessTokenAsync previously did) can hand a stale/expired access token to the pod, - // which the pod then trusts unconditionally (its "fast path" skips its own Key Vault fetch - // whenever a pre-resolved token arrives) — producing GitHubCopilotUnauthorizedException at - // /configure. Routing through the same GetValidAccessTokenAsync used by GitHubCopilotClientFactory - // ensures a near-expiry token is transparently rotated before being handed to a newly-launched pod. - // Null in unit tests → falls back to the raw (non-refreshing) token store read. When present, - // it is authoritative: a null/failed refresh must never fall back to the rejected raw token. - private readonly IGitHubAccessTokenProvider? _accessTokenProvider; - // Replica-safe run secret store used to persist the per-run preview-runner credential so a - // reconcile/keepalive on either API replica can re-fetch it, and to durably DELETE it on pod - // release (spec-006 decouple-preview, BLOCKER A / RESIDUAL). Null in unit tests → no minting. - private readonly ISecretStore? _secretStore; - // Durable run-event log used to emit sandbox.provisioning_pending heartbeats into the CHILD run's - // stream while its AgentHost claim is still being scheduled by Kubernetes (unbound). Keeps the - // parent coordinator's stall timer alive during a legitimately-long Pending wait (issue #217). - // Null in unit tests → the heartbeat is skipped (same null-skip convention as the readiness probe). - private readonly IRunEventStream? _runEventStream; - // Source of the per-run AutoApproveTools flag propagated to the warm pod via /configure (bug - // #221). Null in unit tests → the flag defaults false (same null-skip convention as above). - private readonly IRunOptionsStore? _runOptions; - // First-class preview lifecycle reconciler. ReleaseAgentHostPodAsync derives durable - // Previewable/PreviewActive state and applies all retention or cleanup effects before deciding - // whether to delete the claim. - private readonly Agentweaver.Api.Sandbox.Preview.ISandboxPreviewService? _previewService; - - public bool IsRealIsolation => true; - public string BackendName => "kubernetes-sandbox-claim"; - public string SelectionReason => - "Kubernetes-native sandbox via SandboxClaim warm pool (Kata VM isolation, NetworkPolicy egress restriction)."; - public bool HasNetworkWarning => false; - public string? NetworkWarningMessage => null; - - internal KubernetesSandboxExecutor( - IKubernetes client, - KubernetesSandboxOptions options, - ILogger logger, - IPodNameRegistry? podRegistry = null, - IAgentHostTurnTokenRegistry? turnTokenRegistry = null, - IAgentHostReadinessProbe? readinessProbe = null, - IRunSubmittingUserResolver? submittingUserResolver = null, - IHttpClientFactory? httpClientFactory = null, - IGitHubTokenStore? tokenStore = null, - ISecretStore? secretStore = null, - IRunEventStream? runEventStream = null, - IRunOptionsStore? runOptions = null, - IGitHubAccessTokenProvider? accessTokenProvider = null, - Agentweaver.Api.Sandbox.Preview.ISandboxPreviewService? previewService = null, - IGitHubTokenScopeProvider? tokenScopeProvider = null, - Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null) - { - _client = client; - _options = options; - _logger = logger; - _podRegistry = podRegistry; - _turnTokenRegistry = turnTokenRegistry; - _readinessProbe = readinessProbe; - _submittingUserResolver = submittingUserResolver; - _httpClientFactory = httpClientFactory; - _tokenStore = tokenStore; - _tokenScopeProvider = tokenScopeProvider; - _secretStore = secretStore; - _runEventStream = runEventStream; - _runOptions = runOptions; - _accessTokenProvider = accessTokenProvider; - _previewService = previewService; - _authorshipCapabilityStore = authorshipCapabilityStore; - } - - public async Task ExecuteAsync( - SandboxCommand command, CancellationToken ct = default) - { - // Use the Agentweaver run ID as the claim name when available so the pod can be - // looked up by run ID later (preview port-forward). Fall back to a random ID. - var claimName = string.IsNullOrEmpty(command.AgentweaverRunId) - ? $"run-{Guid.NewGuid():N}"[..20] - : SandboxClaimConventions.DeriveRunCommandClaimName(command.AgentweaverRunId); - - var requestedTimeoutMs = command.TimeoutMs > 0 - ? command.TimeoutMs - : _options.TimeoutSeconds * 1000; - var maxCommandTimeoutMs = Math.Max(1000, (_options.TimeoutSeconds * 1000) - 30_000); - var timeoutMs = Math.Min(requestedTimeoutMs, maxCommandTimeoutMs); - if (timeoutMs < requestedTimeoutMs) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: command timeout clamped from {RequestedMs}ms to {TimeoutMs}ms so it stays below SandboxClaim TTL ({TtlSeconds}s)", - requestedTimeoutMs, timeoutMs, _options.TimeoutSeconds); - } - - string podWorkingDirectory; - try - { - podWorkingDirectory = ResolvePodWorkingDirectory(command.WorkingDirectory); - } - catch (Exception ex) - { - _logger.LogError(ex, - "KubernetesSandboxExecutor: invalid workspace path {WorkingDirectory}; configured mount is {WorkspaceMountPath}", - command.WorkingDirectory, _options.WorkspaceMountPath); - return new SandboxExecResult(1, "", ex.Message, false, false); - } - - _logger.LogInformation( - "KubernetesSandboxExecutor: using workspace path {WorkspacePath} for claim {Claim} (requested {RequestedWorkingDirectory})", - podWorkingDirectory, claimName, command.WorkingDirectory); - - using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct); - linked.CancelAfter(timeoutMs); - var token = linked.Token; - var claimCreated = false; - - try - { - _logger.LogInformation( - "KubernetesSandboxExecutor: creating SandboxClaim {Claim}", claimName); - claimCreated = await CreateClaimAsync(claimName, token); - - var podName = await WaitForBoundAsync(claimName, token); - _logger.LogInformation( - "KubernetesSandboxExecutor: claim {Claim} bound to pod {Pod}", claimName, podName); - - // Register pod name so PortForwardService can locate it by Agentweaver run ID. - // Run-scoped mappings are cleared by run lifecycle cleanup, not per command, so - // preview tunnels can remain available for the whole run while the claim TTL is valid. - if (!string.IsNullOrEmpty(command.AgentweaverRunId)) - _podRegistry?.Register(command.AgentweaverRunId, podName); - - return await ExecInPodAsync(podName, command, podWorkingDirectory, token); - } - catch (OperationCanceledException) when (!ct.IsCancellationRequested) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: timed out waiting for claim {Claim}", claimName); - return new SandboxExecResult(-1, "", "Timed out waiting for sandbox pod.", true, false); - } - finally - { - if (claimCreated && string.IsNullOrEmpty(command.AgentweaverRunId)) - await DeleteClaimAsync(claimName); - else if (claimCreated) - _logger.LogDebug( - "KubernetesSandboxExecutor: retaining SandboxClaim {Claim} for run {RunId} preview until run cleanup or TTL", - claimName, command.AgentweaverRunId); - } - } - - public async IAsyncEnumerable StreamAsync( - SandboxCommand command, - [EnumeratorCancellation] CancellationToken ct = default) - { - var result = await ExecuteAsync(command, ct); - foreach (var line in result.Stdout.Split('\n')) - yield return new SandboxOutputChunk(SandboxOutputStream.Stdout, line); - if (!string.IsNullOrEmpty(result.Stderr)) - foreach (var line in result.Stderr.Split('\n')) - yield return new SandboxOutputChunk(SandboxOutputStream.Stderr, line); - yield return new SandboxOutputChunk(SandboxOutputStream.ExitCode, result.ExitCode.ToString()); - } - - // ── IAgentHostPodLifecycle — pod-per-run lifecycle (spec §9 / Q3) ───────────── - - /// - public Task LaunchAgentHostPodAsync(string runId, CancellationToken ct = default) => - LaunchAgentHostPodAsync(runId, new AgentHostLaunchContext(SharedWorkingDirectory: null), ct); - - /// - public Task LaunchAgentHostPodAsync( - string runId, - string? workingDirectoryOverride, - CancellationToken ct = default) => - LaunchAgentHostPodAsync( - runId, - new AgentHostLaunchContext(SharedWorkingDirectory: workingDirectoryOverride), - ct); - - /// - public async Task LaunchAgentHostPodAsync( - string runId, - AgentHostLaunchContext launchContext, - CancellationToken ct = default) - { - var claimName = SandboxClaimConventions.DeriveAgentHostClaimName(runId); - var requestedWorkingDirectory = string.IsNullOrWhiteSpace(launchContext.SharedWorkingDirectory) - ? null - : Path.GetFullPath(launchContext.SharedWorkingDirectory); - - _logger.LogInformation( - "KubernetesSandboxExecutor: launching AgentHost pod for run {RunId} via claim {Claim}", - runId, claimName); - - // Resolve the run's submitting user so the pod can scope GitHub Copilot auth to that user's - // signed-in token. The user's Key Vault secret name (Option C warm-pool path) is derived here - // and delivered to the pod via /configure — never another user's secret. - var submittingUser = await ResolveSubmittingUserAsync(runId, ct).ConfigureAwait(false); - if (string.IsNullOrWhiteSpace(submittingUser)) - { - throw new InvalidOperationException( - $"Cannot launch AgentHost pod for run '{runId}' without a submitting user; " + - "the /configure call must scope the pod to the run owner's Key Vault token."); - } - - _logger.LogInformation( - "KubernetesSandboxExecutor: resolved submitting user for run {RunId}; will configure pod via /configure.", - runId); - - var (configProjectId, configAgentName) = _submittingUserResolver is not null - ? await _submittingUserResolver.GetRunIdentityAsync(runId, ct).ConfigureAwait(false) - : (null, null); - - // ghtok-user--{base32(userId)} — the SAME mapping the API uses when persisting the token to KV. - // With Entra sign-in the user's credentials live under the ACTIVE linked GitHub identity's - // scope (user-link:{oid}:{login}), so resolve the effective scope rather than assuming the - // legacy per-user scope, which is never written in that mode. - var effectiveScope = _tokenScopeProvider is not null - ? await _tokenScopeProvider - .ResolveAsync(submittingUser!, configProjectId, ct) - .ConfigureAwait(false) - : _tokenStore is IEffectiveGitHubTokenScopeResolver scopeResolver - ? await scopeResolver.ResolveEffectiveScopeAsync(submittingUser!, ct).ConfigureAwait(false) - : GitHubTokenScope.ForUser(submittingUser!); - var kvUserSecretName = KeyVaultSecretStore.SanitizeKey(effectiveScope.Key); - var turnToken = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)); - var claimCreated = false; - try - { - // Bind to the SHARED, pre-warmed AgentHost warm pool (replicas: 2). No per-run SPC, - // SandboxTemplate, or warm pool — the pod is already warm and gets its per-run context - // via the /configure POST below. - claimCreated = await CreateAgentHostClaimAsync( - claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); - - if (!claimCreated && launchContext.Purpose == AgentHostPurpose.OperatorAssistant) - { - // Every operator turn carries the CURRENT browser/platform bearer. An orphaned - // claim from a crashed prior turn is already configured with the old credential - // and /configure is intentionally one-shot, so it must never be reused. - _logger.LogInformation( - "KubernetesSandboxExecutor: recreating existing AgentHost claim {Claim} for a fresh operator-assistant caller credential.", - claimName); - await DeleteClaimAsync(claimName).ConfigureAwait(false); - _podRegistry?.Unregister(runId); - _turnTokenRegistry?.UnregisterTurnToken(runId); - await Task.Delay(1000, ct).ConfigureAwait(false); - claimCreated = await CreateAgentHostClaimAsync( - claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); - if (!claimCreated) - { - throw new InvalidOperationException( - $"AgentHost claim '{claimName}' was deleted to refresh the operator-assistant caller credential, " + - "but the replacement create still conflicted."); - } - } - else if (!claimCreated && launchContext.WorkspaceMode != ExecutionWorkspaceMode.Shared) - { - _logger.LogInformation( - "KubernetesSandboxExecutor: recreating existing AgentHost claim {Claim} for immutable pod-local workspace configuration (mode={Mode}).", - claimName, - launchContext.WorkspaceMode); - await DeleteClaimAsync(claimName).ConfigureAwait(false); - _podRegistry?.Unregister(runId); - _turnTokenRegistry?.UnregisterTurnToken(runId); - await Task.Delay(1000, ct).ConfigureAwait(false); - claimCreated = await CreateAgentHostClaimAsync( - claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); - if (!claimCreated) - { - throw new InvalidOperationException( - $"AgentHost claim '{claimName}' was deleted for immutable pod-local workspace configuration, " + - "but the replacement create still conflicted."); - } - } - else if (!claimCreated && requestedWorkingDirectory is not null) - { - var existingWorkingDirectory = await TryGetAgentHostClaimWorkingDirectoryAsync(claimName, ct) - .ConfigureAwait(false); - var sameWorktree = string.Equals( - existingWorkingDirectory, requestedWorkingDirectory, StringComparison.Ordinal); - var hasTurnToken = !string.IsNullOrWhiteSpace(_turnTokenRegistry?.TryGetTurnToken(runId)); - - if (!sameWorktree || !hasTurnToken) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: existing AgentHost claim {Claim} for run {RunId} " + - "is not reusable (sameWorktree={SameWorktree}, hasTurnToken={HasTurnToken}); recreating.", - claimName, runId, sameWorktree, hasTurnToken); - await DeleteClaimAsync(claimName).ConfigureAwait(false); - _podRegistry?.Unregister(runId); - _turnTokenRegistry?.UnregisterTurnToken(runId); - await Task.Delay(1000, ct).ConfigureAwait(false); - claimCreated = await CreateAgentHostClaimAsync( - claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); - if (!claimCreated) - { - throw new InvalidOperationException( - $"AgentHost claim '{claimName}' for run '{runId}' was deleted for worktree reconfiguration, " + - "but the replacement create still conflicted. Retrying later avoids reusing a token-less or stale pod."); - } - } - } - - var podName = await WaitForBoundWithProvisioningHeartbeatAsync(runId, claimName, ct).ConfigureAwait(false); - _logger.LogInformation( - "KubernetesSandboxExecutor: AgentHost claim {Claim} bound to pod {Pod}", claimName, podName); - - // Register also persists sandbox.execution_pod.bound into the shared RunEvents store so - // graph snapshots/deltas on any API replica can resolve the execution pod. - _podRegistry?.Register(runId, podName); - if (claimCreated) - _turnTokenRegistry?.RegisterTurnToken(runId, turnToken); - - var activeTurnToken = claimCreated - ? turnToken - : _turnTokenRegistry?.TryGetTurnToken(runId); - if (_authorshipCapabilityStore is not null && !string.IsNullOrWhiteSpace(activeTurnToken)) - { - await _authorshipCapabilityStore.RegisterAsync( - runId, activeTurnToken, DateTimeOffset.UtcNow.AddDays(1), ct).ConfigureAwait(false); - } - - var podIp = await GetPodIpAsync(podName, ct).ConfigureAwait(false); - - var endpointUrl = AgentHostEndpoint.Build( - _options.RequireMtls, podIp, _options.AgentHostPort, _options.AgentHostA2APath); - - // A2A cold-start gate: the claim binds when the pod is Running, but the AgentHost Kestrel - // listener takes ~20-30s more to bind :8088. Without this wait the worker's first A2A POST - // hits a closed port → "Connection refused" → the run fails mid-turn. Poll /healthz until the - // app is actually serving so a not-yet-ready pod is a deterministic LAUNCH failure instead. - // NOTE: a warm/standby pod serves /healthz BEFORE /configure (the readiness gate exempts - // /configure), so this confirms reachability prior to injecting the run context. - if (_readinessProbe is not null) - { - var scheme = AgentHostEndpoint.Scheme(_options.RequireMtls); - var readinessUrl = - $"{scheme}://{podIp}:{_options.AgentHostPort}{_options.AgentHostHealthzPath}"; - - _logger.LogInformation( - "KubernetesSandboxExecutor: waiting for AgentHost readiness for run {RunId} at {Url}", - runId, readinessUrl); - - try - { - await _readinessProbe.WaitUntilReadyAsync(readinessUrl, ct).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - throw new InvalidOperationException( - $"AgentHost pod '{podName}' for run '{runId}' did not become ready at {readinessUrl} " + - $"within {_options.AgentHostReadyTimeoutSeconds}s; failing the launch.", ex); - } - } - - // Warm-pool deferred /configure: inject the per-run RunId/UserId/TurnBearerToken and the - // KV secret name into the already-warm pod, which then runs SetupAsync and becomes ready. - // Normal roles use the shared orchestration worktree. Local workspace modes carry - // immutable source refs; AgentHost creates their effective root inside execution-scratch. - if (claimCreated) - { - var effectiveWorkingDirectory = await CallAgentHostConfigureAsync( - podIp, _options.AgentHostPort, runId, submittingUser, turnToken, kvUserSecretName, - effectiveScope, - await ResolveGitHubAccessTokenAsync(effectiveScope, submittingUser, ct).ConfigureAwait(false), - requestedWorkingDirectory ?? await ResolveWorkingDirectoryAsync(runId, ct).ConfigureAwait(false), - launchContext, - configProjectId, - configAgentName, - ct) - .ConfigureAwait(false); - if (!string.IsNullOrWhiteSpace(effectiveWorkingDirectory)) - _podRegistry?.RegisterEffectiveWorkingDirectory(runId, effectiveWorkingDirectory); - } - else - { - _logger.LogInformation( - "KubernetesSandboxExecutor: reusing already-configured AgentHost claim {Claim} for run {RunId}", - claimName, runId); - } - - _podRegistry?.RegisterAgentEndpoint(runId, endpointUrl); - - _logger.LogInformation( - "KubernetesSandboxExecutor: AgentHost A2A endpoint for run {RunId} = {Endpoint}", - runId, endpointUrl); - - return endpointUrl; - } - catch - { - if (claimCreated) - await DeleteClaimAsync(claimName).ConfigureAwait(false); - _podRegistry?.Unregister(runId); - _turnTokenRegistry?.UnregisterTurnToken(runId); - if (_authorshipCapabilityStore is not null) - { - await _authorshipCapabilityStore.RemoveAsync(runId, CancellationToken.None) - .ConfigureAwait(false); - } - // Crash/timeout during launch: delete any credential minted before the failure so it is - // never left behind (spec-006 decouple-preview, RESIDUAL rev3 gap). - await DeletePreviewRunnerCredentialAsync(runId, CancellationToken.None).ConfigureAwait(false); - throw; - } - } - - /// - public async Task ReleaseAgentHostPodAsync(string runId, CancellationToken ct = default) - { - var claimName = SandboxClaimConventions.DeriveAgentHostClaimName(runId); - - // Issue #542: if a live preview is still active for this run, releasing the pod here (at the - // originating subtask's turn end) would 404 the preview URL before any human-review gate or - // demo viewer can open it. Defer the claim delete while the preview is alive; the preview's own - // idle/max expiry + the reaper will eventually reap the pod, so this cannot leak. - if (_previewService is not null && - await _previewService.ReconcilePreviewLifecycleAsync(runId, ct).ConfigureAwait(false) - == Agentweaver.Api.Sandbox.Preview.PreviewLifecycleState.PreviewActive) - { - _logger.LogInformation( - "KubernetesSandboxExecutor: deferring AgentHost pod release for run {RunId} (claim " + - "{Claim}) — a live preview is still active; the preview idle/max expiry will reap it.", - runId, claimName); - return; - } - - _logger.LogInformation( - "KubernetesSandboxExecutor: releasing AgentHost pod for run {RunId} (claim {Claim})", - runId, claimName); - - await DeleteClaimAsync(claimName, ct).ConfigureAwait(false); - _podRegistry?.Unregister(runId); - _turnTokenRegistry?.UnregisterTurnToken(runId); - if (_authorshipCapabilityStore is not null) - await _authorshipCapabilityStore.RemoveAsync(runId, ct).ConfigureAwait(false); - await DeletePreviewRunnerCredentialAsync(runId, ct).ConfigureAwait(false); - - _logger.LogInformation( - "KubernetesSandboxExecutor: AgentHost pod released for run {RunId}", runId); - } - - /// - /// Resolves the submitting user for via the injected resolver, never - /// throwing (a lookup failure must not fail the launch — it degrades to omitting the user id). - /// - private async Task ResolveSubmittingUserAsync(string runId, CancellationToken ct) - { - if (_submittingUserResolver is null) - return null; - - try - { - return await _submittingUserResolver.GetSubmittingUserAsync(runId, ct).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger.LogWarning( - ex, - "KubernetesSandboxExecutor: failed to resolve submitting user for run {RunId}; " + - "AgentHost__UserId will be omitted.", - runId); - return null; - } - } - - /// - /// Resolves the per-run working directory (shared orchestration worktree path) for - /// via the injected resolver, never throwing (a lookup failure must not - /// fail the launch — it degrades to omitting the working directory, so the pod falls back to its - /// static AgentHost__WorkingDirectory env default). - /// - private async Task ResolveWorkingDirectoryAsync(string runId, CancellationToken ct) - { - if (_submittingUserResolver is null) - return null; - - try - { - return await _submittingUserResolver.GetWorkingDirectoryAsync(runId, ct).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger.LogWarning( - ex, - "KubernetesSandboxExecutor: failed to resolve working directory for run {RunId}; " + - "AgentHost__WorkingDirectory env default will be used.", - runId); - return null; - } - } - /// (AgentHostWarmPoolRef, replicas: 2). No spec.env is injected — the v0.5.0 - /// controller bypasses warm pool adoption whenever spec.env or - /// spec.volumeClaimTemplates are present. All static config lives in the SandboxTemplate - /// or agenthost-config ConfigMap. The per-run context (RunId / UserId / TurnBearerToken / - /// KV secret name) is delivered after bind via POST /configure - /// (). - /// - private async Task CreateAgentHostClaimAsync( - string claimName, string warmPoolName, string? workingDirectory, string runId, CancellationToken ct) - { - var annotations = new Dictionary - { - // Persist the ORIGINAL run id so the reaper can recover it from an orphaned claim (the - // claim name is a lossy 12-char derivation) and delete run-scoped side artifacts such as - // the per-run preview-runner credential (spec-006 decouple-preview). - [SandboxClaimConventions.RunIdAnnotation] = runId, - }; - if (!string.IsNullOrWhiteSpace(workingDirectory)) - annotations["agentweaver.io/working-directory"] = workingDirectory; - - var manifest = new - { - apiVersion = $"{ApiGroup}/{ApiVersion}", - kind = "SandboxClaim", - metadata = new - { - name = claimName, - @namespace = _options.Namespace, - annotations = annotations.Count == 0 ? null : annotations, - }, - spec = new - { - // v0.5.0 v1beta1 SandboxClaimSpec: spec.warmPoolRef.name references the - // SandboxWarmPool to bind from. sandboxTemplateRef+warmpool were the - // v0.4.x/v1alpha1 deprecated fields. - warmPoolRef = new { name = warmPoolName }, - lifecycle = new { ttlSecondsAfterFinished = _options.TimeoutSeconds, shutdownPolicy = "Delete" }, - }, - }; - - // Idempotent create with bounded transient-fault retry (issue #230). A mid-flight connection - // reset can commit the SandboxClaim server-side BEFORE we observe the response, so the retry - // may see a 409 for OUR OWN create — handled attempt-awarely below. - for (var attempt = 1; ; attempt++) - { - try - { - await _client.CustomObjects.CreateNamespacedCustomObjectAsync( - manifest, ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, - cancellationToken: ct).ConfigureAwait(false); - return true; - } - catch (HttpOperationException ex) when (ex.Response?.StatusCode == System.Net.HttpStatusCode.Conflict) - { - if (attempt > 1) - { - // Retry-409: a transient reset committed our create server-side before we saw the - // response, and this retry now observes our own claim. We own it → return true so - // the caller registers the turn token and runs /configure exactly as on a 200, - // rather than taking the silent "reuse pre-existing claim" path (which would leave - // the pod un-configured and token-less). - _logger.LogInformation( - "KubernetesSandboxExecutor: SandboxClaim {Claim} returned 409 on retry attempt {Attempt}; " + - "treating as our own create that committed before a transient reset — configuring it.", - claimName, attempt); - return true; - } - - // First-attempt 409: a genuinely pre-existing claim owned by an earlier launch. - _logger.LogInformation( - "KubernetesSandboxExecutor: SandboxClaim {Claim} already exists; waiting for existing claim", - claimName); - return false; - } - catch (Exception ex) when (attempt < MaxK8sAttempts && IsTransientK8sFault(ex, ct)) - { - var delay = BackoffWithJitter(attempt); - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: transient fault creating SandboxClaim {Claim} on attempt " + - "{Attempt}/{Max}; retrying in {DelayMs}ms.", - claimName, attempt, MaxK8sAttempts, (int)delay.TotalMilliseconds); - await Task.Delay(delay, ct).ConfigureAwait(false); - } - } - } - - // ── Transient Kubernetes API resilience (issue #230) ────────────────────────── - - /// - /// Executes an idempotent Kubernetes API call with a bounded retry ( - /// total attempts) over transient faults only — a mid-flight connection reset - /// (SocketException 104 → IOException → HttpRequestException), a 429/5xx from the API server, or an - /// HttpClient timeout. Caller cancellation is never retried and aborts the backoff immediately - /// (await Task.Delay(delay, ct)). Non-transient faults (e.g. 404/409/422) propagate on the - /// first attempt. MUST NOT wrap non-idempotent calls (e.g. the AgentHost POST /configure, - /// whose second delivery 409-hard-fails). - /// - private async Task ExecuteK8sWithRetryAsync( - Func> operation, CancellationToken ct) - { - for (var attempt = 1; ; attempt++) - { - try - { - return await operation(ct).ConfigureAwait(false); - } - catch (Exception ex) when (attempt < MaxK8sAttempts && IsTransientK8sFault(ex, ct)) - { - var delay = BackoffWithJitter(attempt); - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: transient Kubernetes API fault on attempt {Attempt}/{Max}; " + - "retrying in {DelayMs}ms.", attempt, MaxK8sAttempts, (int)delay.TotalMilliseconds); - await Task.Delay(delay, ct).ConfigureAwait(false); - } - } - } - - /// - /// Exponential backoff (~250ms · 2^(attempt-1), capped at ~2s) plus 0-250ms jitter to de-sync - /// concurrent launches retrying against the same API server after a blip. - /// - private static TimeSpan BackoffWithJitter(int attempt) - { - var baseMs = Math.Min(250 * (1 << (attempt - 1)), 2000); - var jitterMs = Random.Shared.Next(0, 250); - return TimeSpan.FromMilliseconds(baseMs + jitterMs); - } - - /// - /// True only for faults worth retrying an idempotent k8s call over: 429/5xx from the API server, - /// a socket/IO connection reset (directly or nested in an inner exception), or an HttpClient - /// timeout ( with no caller cancellation). Caller - /// cancellation short-circuits to false so a genuine cancel is never retried. A 409 Conflict is - /// intentionally NOT transient here — it is handled separately (idempotent create semantics). - /// - private static bool IsTransientK8sFault(Exception ex, CancellationToken ct) - { - if (ct.IsCancellationRequested) return false; // caller cancel — never retry - switch (ex) - { - case HttpOperationException k when k.Response is not null: - var s = (int)k.Response.StatusCode; - return s == 429 || s >= 500; // 409 handled separately, NOT here - case HttpRequestException: return true; - case IOException: return true; - case OperationCanceledException: // includes TaskCanceledException (HttpClient timeout) - return !ct.IsCancellationRequested; - } - for (Exception? i = ex.InnerException; i is not null; i = i.InnerException) - if (i is SocketException or IOException) return true; - return false; - } - - private async Task TryGetAgentHostClaimWorkingDirectoryAsync(string claimName, CancellationToken ct) - { - try - { - var raw = await _client.CustomObjects.GetNamespacedCustomObjectAsync( - ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, - cancellationToken: ct).ConfigureAwait(false); - var json = JsonSerializer.Serialize(raw); - using var doc = JsonDocument.Parse(json); - if (doc.RootElement.TryGetProperty("metadata", out var meta) && - meta.TryGetProperty("annotations", out var ann) && - ann.TryGetProperty("agentweaver.io/working-directory", out var wd) && - wd.ValueKind == JsonValueKind.String) - return wd.GetString(); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to read working-directory annotation for claim {Claim}", - claimName); - } - - return null; - } - - /// - /// Resolves the run owner's GitHub access token from the API-side token store so it can be - /// forwarded in the /configure body. The kata VM pod cannot reach Azure AD or Key Vault - /// (Cilium FQDN policies use eBPF interception that doesn't cross the guest kernel boundary). - /// Never throws — a lookup failure degrades gracefully: the pod will attempt the KV fetch itself - /// (which may fail) rather than causing a hard launch failure here. - /// - private async Task ResolveGitHubAccessTokenAsync( - GitHubTokenScope scope, - string userId, - CancellationToken ct) - { - // Prefer the refresh-aware provider (issue #523): a fresh AgentHost pod launched late in a - // long-running assembly (e.g. the Build & Test gate, well after the run's earlier subtask - // stages) can be handed a near-expiry or already-expired access token if we only ever read - // the raw stored entry — the pod's "fast path" trusts a pre-resolved token unconditionally - // and never re-validates it against Key Vault or GitHub. Routing through - // GetValidAccessTokenAsync mirrors GitHubCopilotClientFactory.CreateClientAsync and - // transparently rotates the token before it is handed to the pod. - if (_accessTokenProvider is not null) - { - try - { - var refreshed = await _accessTokenProvider.GetValidAccessTokenAsync(scope, ct) - .ConfigureAwait(false); - if (string.IsNullOrEmpty(refreshed)) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: refresh-aware GitHub token provider returned no valid credential " + - "for {UserId} (scope {Scope}); refusing raw-token fallback.", - userId, - scope.Key); - } - return refreshed; - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to resolve/refresh GitHub token for {UserId} via " + - "IGitHubAccessTokenProvider (scope {Scope}); refusing raw-token fallback.", - userId, - scope.Key); - return null; - } - } - - if (_tokenStore is null) - return null; - - try - { - var entry = await _tokenStore.GetAsync(scope, ct).ConfigureAwait(false); - if (entry.Status == GitHubTokenStatus.SignedIn && !string.IsNullOrEmpty(entry.AccessToken)) - return entry.AccessToken; - } - catch (Exception ex) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to pre-resolve GitHub token for {UserId} — pod will fall back to KV.", - userId); - } - - return null; - } - - /// - /// Injects the per-run context into an already-warm AgentHost pod via its one-time - /// POST /configure endpoint. The pod then fetches ONLY - /// from Key Vault (its configured user's token) and runs SetupAsync. The endpoint is guarded by - /// NetworkPolicy (ingress to AgentHost pods restricted to API/worker), not the TurnBearerToken - /// (which is itself delivered here). Idempotency: a second call returns 409 and is treated as a - /// hard launch failure. - /// - private async Task CallAgentHostConfigureAsync( - string podIp, int port, string runId, string userId, string turnBearerToken, - string kvUserSecretName, GitHubTokenScope tokenScope, string? gitHubAccessToken, - string? sharedWorkingDirectory, - AgentHostLaunchContext launchContext, - string? projectId, - string? agentName, - CancellationToken ct) - { - if (_httpClientFactory is null) - { - // No HttpClient available (unit tests). Mirrors the readiness-probe null-skip; in-cluster - // the factory is always present, so this never short-circuits a real launch. - _logger.LogWarning( - "KubernetesSandboxExecutor: no IHttpClientFactory — skipping /configure for run {RunId}.", - runId); - return null; - } - - var scheme = AgentHostEndpoint.Scheme(_options.RequireMtls); - var configureUrl = $"{scheme}://{podIp}:{port}/configure"; - - // Mint a FRESH per-run preview-runner credential (spec-006 decouple-preview, BLOCKER A). - // Delivered in-memory via this /configure body ONLY (never pod env/file), and persisted to the - // run secret store so any replica can re-fetch it for reconcile/keepalive. Durably deleted on - // pod release. Every launch/relaunch mints a new value — the old one is never reused. - var previewRunnerCredential = await MintPreviewRunnerCredentialAsync(runId, ct).ConfigureAwait(false); - - var body = new - { - runId, - userId, - turnBearerToken, - kvUserSecretName, - gitHubAccessToken, - callerBearerToken = launchContext.CallerBearerToken, - // Keep the legacy property during rolling upgrades; new AgentHosts prefer the explicit - // sharedWorkingDirectory descriptor and create any local workspace inside the pod. - workingDirectory = sharedWorkingDirectory, - sharedWorkingDirectory, - previewRunnerCredential, - purpose = launchContext.Purpose.ToString(), - launchContext.SourceRepositoryPath, - launchContext.SourceRef, - launchContext.BaseCommitSha, - launchContext.ExpectedTreeHash, - workspaceMode = launchContext.WorkspaceMode.ToString(), - launchContext.ScratchRoot, - launchContext.CommitAuthorName, - launchContext.CommitAuthorEmail, - // Per-run AutoApproveTools flag (bug #221). Resolved from the API-side run-options store - // keyed by the child runId; defaults false when the store is unavailable (unit tests). - autoApproveTools = _runOptions?.Get(runId).AutoApproveTools ?? false, - // Per-run project/agent identity (#335). Delivered so the in-pod agent's tool schema - // includes the Agentweaver API tools (record_memory, get_memory, submit_decision, - // list_decisions, list_inbox). Warm pods boot with an empty static AgentHost__ProjectId - // /AgentName, so without these the memory/decision tools never reach the agent. - projectId, - agentName, - }; - - _logger.LogInformation( - "KubernetesSandboxExecutor: configuring AgentHost pod for run {RunId} at {Url}", - runId, configureUrl); - - using var client = _httpClientFactory.CreateClient(HttpAgentHostReadinessProbe.HttpClientName); - using var response = await client - .PostAsJsonAsync(configureUrl, body, ct) - .ConfigureAwait(false); - var detail = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); - - if (!response.IsSuccessStatusCode) - { - var reason = "agenthost_configure_failed"; - try - { - using var document = JsonDocument.Parse(detail); - if (document.RootElement.TryGetProperty("error", out var error) - && error.ValueKind == JsonValueKind.String - && !string.IsNullOrWhiteSpace(error.GetString())) - reason = error.GetString()!; - } - catch (JsonException) - { - // Plain-text legacy errors keep the generic typed reason. - } - - if (string.Equals( - reason, - "agenthost_configure_copilot_unauthorized", - StringComparison.Ordinal) && - _accessTokenProvider is not null) - { - var refreshed = await _accessTokenProvider - .RefreshAfterUnauthorizedAsync(tokenScope, gitHubAccessToken, ct) - .ConfigureAwait(false); - if (!string.IsNullOrWhiteSpace(refreshed) && - !string.Equals(refreshed, gitHubAccessToken, StringComparison.Ordinal)) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: AgentHost /configure rejected the Copilot credential for run {RunId}; " + - "scope {Scope} was refreshed and the pod must be recreated (recoveryAttempt=1, maxRecoveryAttempts=1).", - runId, - tokenScope.Key); - throw new AgentHostConfigureException( - "agenthost_configure_copilot_token_refreshed", - $"AgentHost /configure rejected the Copilot credential for run '{runId}'. " + - "The credential was refreshed; recreate the one-time-configured pod and retry once.", - (int)response.StatusCode, - retryable: true, - recoveryAction: "recreate_pod_with_refreshed_credential"); - } - - _logger.LogWarning( - "KubernetesSandboxExecutor: AgentHost /configure rejected the Copilot credential for run {RunId}; " + - "scope {Scope} could not produce a different refreshed credential, so the failure is not retryable.", - runId, - tokenScope.Key); - } - - throw new AgentHostConfigureException( - reason, - $"AgentHost /configure for run '{runId}' failed: HTTP {(int)response.StatusCode} {detail}", - (int)response.StatusCode); - } - - if (string.IsNullOrWhiteSpace(detail)) - return null; - - try - { - using var document = JsonDocument.Parse(detail); - if (document.RootElement.TryGetProperty("effectiveWorkingDirectory", out var path) - && path.ValueKind == JsonValueKind.String - && !string.IsNullOrWhiteSpace(path.GetString())) - { - return path.GetString(); - } - } - catch (JsonException ex) - { - _logger.LogWarning( - ex, - "KubernetesSandboxExecutor: AgentHost /configure for run {RunId} returned an invalid success body; preview will use the shared working directory.", - runId); - } - - return null; - } - - /// - /// Mints and persists a fresh per-run preview-runner credential and returns it for in-memory - /// delivery via /configure. Returns when no secret store is - /// available (unit tests) — the pod then relies on the turn token only. The persisted key is - /// derived deterministically from the run id () - /// so the release-time delete matches (spec-006 decouple-preview, BLOCKER A). - /// - private async Task MintPreviewRunnerCredentialAsync(string runId, CancellationToken ct) - { - if (_secretStore is null) - return string.Empty; - - var credential = Preview.PreviewRunnerCredential.Mint(); - var key = Preview.PreviewRunnerCredential.SecretKey(runId); - try - { - await _secretStore.SetSecretAsync(key, credential, etag: null, ct).ConfigureAwait(false); - _logger.LogInformation( - "KubernetesSandboxExecutor: minted per-run preview-runner credential for run {RunId}", runId); - return credential; - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - // Best-effort: a persist failure must not fail the launch. The pod still receives the - // credential in-memory (same-process affinity uses the turn token anyway), but a - // cross-replica reconcile could not re-fetch it — acceptable degradation. - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to persist preview-runner credential for run {RunId}; " + - "delivering in-memory only.", runId); - return credential; - } - } - - /// - /// Durably deletes the per-run preview-runner credential from the run secret store. No-op when - /// absent ( ignores a missing key). Never throws — - /// a delete failure must not break terminal cleanup. Called on EVERY terminal path (happy - /// release + crash/timeout/failed-run via the pod-release seam) so the credential's durable - /// lifetime is bounded by the pod's (spec-006 decouple-preview, RESIDUAL rev3 gap). - /// - private async Task DeletePreviewRunnerCredentialAsync(string runId, CancellationToken ct) - { - if (_secretStore is null) - return; - - try - { - await _secretStore.DeleteSecretAsync(Preview.PreviewRunnerCredential.SecretKey(runId), ct) - .ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to delete preview-runner credential for run {RunId} (best-effort)", - runId); - } - } - - /// - /// Waits for the AgentHost SandboxClaim to bind while emitting periodic - /// heartbeats into the CHILD run's event - /// stream. Scheduling is Kubernetes' job: a claim may sit unbound (pod Pending) for a while until - /// a node frees up or the pool autoscales — that is FINE and must not fail the run (issue #217). - /// The heartbeat keeps the parent coordinator's subtask-stall timer alive during that legitimate - /// wait, mirroring the #212 tool.approval_pending heartbeat. Best-effort: if no - /// is wired (unit tests) this degrades to a plain - /// . - /// - private async Task WaitForBoundWithProvisioningHeartbeatAsync( - string runId, string claimName, CancellationToken ct) - { - if (_runEventStream is null) - return await WaitForBoundAsync(claimName, ct).ConfigureAwait(false); - - var boundTask = WaitForBoundAsync(claimName, ct); - while (true) - { - var delayTask = Task.Delay(SandboxProvisioningHeartbeatInterval, ct); - var completed = await Task.WhenAny(boundTask, delayTask).ConfigureAwait(false); - if (ReferenceEquals(completed, boundTask)) - return await boundTask.ConfigureAwait(false); // propagates the bound pod name / any error - - // The claim is still unbound after the heartbeat interval — emit a non-terminal - // heartbeat so the coordinator's stall window resets while Kubernetes schedules the pod. - await delayTask.ConfigureAwait(false); // observe cancellation - await EmitProvisioningPendingAsync(runId, claimName, ct).ConfigureAwait(false); - } - } - - /// - /// Appends a single heartbeat to - /// 's durable event stream. Best-effort: a stream-append failure is - /// logged and swallowed so it can never fail a launch that Kubernetes would otherwise admit. - /// - private async Task EmitProvisioningPendingAsync(string runId, string claimName, CancellationToken ct) - { - try - { - await _runEventStream!.AppendAsync(runId, new RunEvent(0, EventTypes.SandboxProvisioningPending, new - { - claimName, - timestamp_utc = DateTimeOffset.UtcNow.ToString("O"), - }), ct).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to emit sandbox.provisioning_pending heartbeat for run {RunId} (best-effort)", - runId); - } - } - - /// - /// Parses a Kubernetes CPU quantity into whole cores. Handles plain cores ("24", - /// "1.5") and the millicore suffix ("500m" = 0.5 cores). Returns - /// for an unrecognized format. - /// - internal static bool TryParseCpu(string? value, out double cores) - { - cores = 0; - if (string.IsNullOrWhiteSpace(value)) - return false; - - value = value.Trim(); - if (value.EndsWith("m", StringComparison.Ordinal)) - { - var millis = value[..^1]; - if (double.TryParse(millis, System.Globalization.NumberStyles.Float, - System.Globalization.CultureInfo.InvariantCulture, out var m)) - { - cores = m / 1000.0; - return true; - } - return false; - } - - return double.TryParse(value, System.Globalization.NumberStyles.Float, - System.Globalization.CultureInfo.InvariantCulture, out cores); - } - - /// - /// Reads the pod IP from the Kubernetes API after the claim is Bound. - /// Polls every 2 s until status.podIP is non-empty (pod has been scheduled - /// and assigned a network address). - /// - private async Task GetPodIpAsync(string podName, CancellationToken ct) - { - while (true) - { - ct.ThrowIfCancellationRequested(); - - var pod = await ExecuteK8sWithRetryAsync( - token => _client.CoreV1.ReadNamespacedPodAsync( - podName, _options.Namespace, cancellationToken: token), - ct).ConfigureAwait(false); - - var ip = pod?.Status?.PodIP; - if (!string.IsNullOrWhiteSpace(ip)) - return ip; - - _logger.LogDebug( - "KubernetesSandboxExecutor: waiting for pod IP of {Pod} (current: {Ip})", - podName, ip ?? "(none)"); - - await Task.Delay(2000, ct).ConfigureAwait(false); - } - } - - // ── Claim management ────────────────────────────────────────────────────────── - - private async Task CreateClaimAsync(string claimName, CancellationToken ct) - { - // The cluster service CIDR must be present in SandboxEgressCidrExclusions so - // sandbox NetworkPolicy does not accidentally allow in-cluster service egress. - var manifest = new - { - apiVersion = $"{ApiGroup}/{ApiVersion}", - kind = "SandboxClaim", - metadata = new { name = claimName, @namespace = _options.Namespace }, - spec = new - { - // v0.5.0 v1beta1 SandboxClaimSpec: spec.warmPoolRef.name references the - // SandboxWarmPool to bind from. sandboxTemplateRef+warmpool were the - // v0.4.x/v1alpha1 deprecated fields. - warmPoolRef = new { name = _options.WarmPoolRef }, - lifecycle = new { ttlSecondsAfterFinished = _options.TimeoutSeconds, shutdownPolicy = "Delete" }, - }, - }; - - try - { - await _client.CustomObjects.CreateNamespacedCustomObjectAsync( - manifest, ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, - cancellationToken: ct).ConfigureAwait(false); - return true; - } - catch (HttpOperationException ex) when (ex.Response?.StatusCode == System.Net.HttpStatusCode.Conflict) - { - _logger.LogInformation( - "KubernetesSandboxExecutor: SandboxClaim {Claim} already exists; waiting for existing claim", - claimName); - return false; - } - } - - /// - /// Polls every 2 s until the claim's Ready condition is True; returns the bound - /// pod name from status.sandbox.name. - /// - private async Task WaitForBoundAsync(string claimName, CancellationToken ct) - { - while (true) - { - ct.ThrowIfCancellationRequested(); - - var raw = await ExecuteK8sWithRetryAsync( - token => _client.CustomObjects.GetNamespacedCustomObjectAsync( - ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, - cancellationToken: token), - ct).ConfigureAwait(false); - - var json = JsonSerializer.Serialize(raw); - using var doc = JsonDocument.Parse(json); - - // Surface a controller reconcile failure (e.g. "exceeded quota") as a deterministic - // launch failure with a precise reason instead of polling until the caller times out. - var reconcilerError = SandboxClaimConventions.TryGetReconcilerError(doc.RootElement); - if (reconcilerError is not null) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: claim {Claim} reconcile failed: {Error}", - claimName, reconcilerError); - throw new AgentHostPodReconcilerErrorException( - $"SandboxClaim '{claimName}' could not be provisioned: {reconcilerError}"); - } - - var podName = SandboxClaimConventions.TryGetBoundPodName(doc.RootElement); - if (!string.IsNullOrEmpty(podName)) - return podName; - - await Task.Delay(2000, ct); - } - } - - private async Task DeleteClaimAsync(string claimName, CancellationToken ct = default) - { - try - { - await _client.CustomObjects.DeleteNamespacedCustomObjectAsync( - ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, cancellationToken: ct); - _logger.LogInformation( - "KubernetesSandboxExecutor: deleted claim {Claim}", claimName); - } - catch (Exception ex) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: could not delete claim {Claim} (best-effort)", claimName); - } - } - - // ── Command execution ───────────────────────────────────────────────────────── - - private async Task ExecInPodAsync( - string podName, SandboxCommand command, string podWorkingDirectory, CancellationToken ct) - { - const int maxOutputBytes = 4 * 1024 * 1024; - - var shellScript = BuildShellScript(command, podWorkingDirectory); - - var ws = await _client.WebSocketNamespacedPodExecAsync( - podName, _options.Namespace, - new[] { "/bin/sh", "-c", shellScript }, - container: ContainerName, - stdin: false, stdout: true, stderr: true, tty: false, - cancellationToken: ct); - - using var demux = new StreamDemuxer(ws, StreamType.RemoteCommand); - demux.Start(); - - using var stdoutStream = demux.GetStream(ChannelIndex.StdOut, null); - using var stderrStream = demux.GetStream(ChannelIndex.StdErr, null); - // Channel 3 (Error) carries the terminal v1.Status payload with the real exit code. - using var statusStream = demux.GetStream(ChannelIndex.Error, null); - - var stdoutTask = ReadBoundedAsync(stdoutStream, maxOutputBytes, ct); - var stderrTask = ReadBoundedAsync(stderrStream, maxOutputBytes, ct); - var statusTask = ReadBoundedAsync(statusStream, maxOutputBytes, ct); - - await Task.WhenAll(stdoutTask, stderrTask, statusTask); - - var (stdoutBytes, stdoutTruncated) = await stdoutTask; - var (stderrBytes, stderrTruncated) = await stderrTask; - var (statusBytes, _) = await statusTask; - - var stdout = SandboxOutputRedactor.Default.Redact(Encoding.UTF8.GetString(stdoutBytes)); - var stderr = SandboxOutputRedactor.Default.Redact(Encoding.UTF8.GetString(stderrBytes)); - var exitCode = ParseExitCode(Encoding.UTF8.GetString(statusBytes)); - - return new SandboxExecResult( - exitCode, stdout, stderr, false, stdoutTruncated || stderrTruncated); - } - - /// - /// Reads up to from a stream, stopping at the cap. - /// Returns the bytes collected and whether the output was truncated. - /// - private static async Task<(byte[] Bytes, bool Truncated)> ReadBoundedAsync( - Stream stream, int maxBytes, CancellationToken ct) - { - using var buffer = new MemoryStream(); - var chunk = new byte[8192]; - bool truncated = false; - int read; - while ((read = await stream.ReadAsync(chunk, ct)) > 0) - { - int remaining = maxBytes - (int)buffer.Length; - if (remaining <= 0) { truncated = true; break; } - int take = Math.Min(read, remaining); - buffer.Write(chunk, 0, take); - if (take < read) { truncated = true; break; } - } - return (buffer.ToArray(), truncated); - } - - /// - /// Parses the terminal v1.Status JSON emitted on channel 3. - /// status: "Success" → exit 0. status: "Failure" → the ExitCode - /// cause from details.causes (defaulting to 1 if not present). - /// - private static int ParseExitCode(string statusJson) - { - if (string.IsNullOrWhiteSpace(statusJson)) - return 0; - - try - { - using var doc = JsonDocument.Parse(statusJson); - var root = doc.RootElement; - - var status = root.TryGetProperty("status", out var s) ? s.GetString() : null; - if (string.Equals(status, "Success", StringComparison.OrdinalIgnoreCase)) - return 0; - - if (root.TryGetProperty("details", out var details) && - details.TryGetProperty("causes", out var causes) && - causes.ValueKind == JsonValueKind.Array) - { - foreach (var cause in causes.EnumerateArray()) - { - var reason = cause.TryGetProperty("reason", out var r) ? r.GetString() : null; - if (string.Equals(reason, "ExitCode", StringComparison.OrdinalIgnoreCase) && - cause.TryGetProperty("message", out var m) && - int.TryParse(m.GetString(), out var code)) - return code; - } - } - - // Failure status with no parseable ExitCode cause → non-zero. - return 1; - } - catch (JsonException) - { - return 0; - } - } - - private string ResolvePodWorkingDirectory(string requestedWorkingDirectory) - { - var mountPath = NormalizeUnixPath(_options.WorkspaceMountPath, forceAbsolute: true); - if (string.IsNullOrWhiteSpace(requestedWorkingDirectory)) - return mountPath; - - var requested = NormalizeUnixPath(requestedWorkingDirectory, forceAbsolute: false); - if (IsSameOrChildPath(requested, mountPath)) - return requested; - - throw new InvalidOperationException( - $"Kubernetes sandbox working directory '{requestedWorkingDirectory}' is not under mounted workspace '{mountPath}'. " + - "Configure Workspace:PersistentVolume:MountRoot/Workspace:Path to match the workspace PVC mount used by sandbox pods."); - } - - private static bool IsSameOrChildPath(string path, string root) => - string.Equals(path, root, StringComparison.Ordinal) - || (root == "/" && path.StartsWith("/", StringComparison.Ordinal)) - || path.StartsWith(root + "/", StringComparison.Ordinal); - - private static string NormalizeUnixPath(string path, bool forceAbsolute) - { - var normalized = path.Trim().Replace('\\', '/'); - while (normalized.Contains("//", StringComparison.Ordinal)) - normalized = normalized.Replace("//", "/", StringComparison.Ordinal); - if (forceAbsolute && !normalized.StartsWith("/", StringComparison.Ordinal)) - normalized = "/" + normalized; - return normalized.Length > 1 ? normalized.TrimEnd('/') : normalized; - } - - private static string BuildShellScript(SandboxCommand command, string podWorkingDirectory) - { - var sb = new StringBuilder(); - - if (command.Environment is { Count: > 0 }) - { - foreach (var (key, value) in command.Environment) - sb.AppendLine($"export {key}={ShellSingleQuote(value)}"); - } - - sb.AppendLine($"cd {ShellSingleQuote(podWorkingDirectory)}"); - - sb.Append(command.CommandLine); - return sb.ToString(); - } - - private static string ShellSingleQuote(string s) => - "'" + s.Replace("'", "'\\''") + "'"; -} +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; +using System.Net.Http.Json; +using System.Net.Sockets; +using System.Text.Json; +using Agentweaver.Api.Auth; +using Agentweaver.Api.Infrastructure; +using Agentweaver.AgentRuntime.Workflow; +using Agentweaver.Domain; +using k8s; +using k8s.Autorest; +using Agentweaver.SandboxExec; +using Microsoft.Extensions.Logging; + +namespace Agentweaver.Api.Sandbox; + +/// +/// Configures the Kubernetes SandboxClaim backend. +/// Bound from the Sandbox:Kubernetes configuration section. +/// +public sealed class KubernetesSandboxOptions +{ + public string Namespace { get; init; } = "agentweaver"; + public string TemplateRef { get; init; } = "agentweaver-sandbox"; + /// + /// SandboxWarmPool the generic command-exec claim binds to. In the v1beta1 CRD a + /// SandboxClaim references a SandboxWarmPool (spec.warmPoolRef.name), + /// which in turn references the SandboxTemplate. Default: agentweaver-sandbox. + /// + public string WarmPoolRef { get; init; } = "agentweaver-sandbox"; + /// Path where the shared workspace PVC is mounted inside API and sandbox pods. + public string WorkspaceMountPath { get; init; } = "/workspace"; + /// SandboxClaim TTL. Command timeouts are capped below this so controller GC cannot interrupt exec. + public int TimeoutSeconds { get; init; } = 600; + /// Cluster service CIDR that must be excluded by sandbox egress policy. + public string? ServiceCidr { get; init; } + public IReadOnlyList SandboxEgressCidrExclusions { get; init; } = []; + + // ── Pod-per-run AgentHost lifecycle options (spec §9 / Q3 hybrid) ───────── + + /// + /// SandboxWarmPool the AgentHost (pod-per-run) claim binds to in the v0.5.0 v1beta1 CRD + /// (spec.warmPoolRef.name). The pool itself references the AgentHost SandboxTemplate. + /// Default: agentweaver-agent-host. + /// + public string AgentHostWarmPoolRef { get; init; } = "agentweaver-agent-host"; + + /// + /// Port the AgentHost Kestrel listener binds to inside the pod. + /// Worker builds the A2A endpoint as http://<podIP>:<AgentHostPort><AgentHostA2APath>. + /// TLS/mTLS termination is owned by Link (H1) — leave hook here for cert wiring. + /// Default: 8088. + /// + public int AgentHostPort { get; init; } = 8088; + + /// + /// A2A path prefix mounted by MapA2AHttpJson inside the AgentHost pod. + /// Must match AgentHost:A2APath set in the pod's configuration. + /// Default: /a2a/agent. + /// + public string AgentHostA2APath { get; init; } = "/a2a/agent"; + + /// + /// When (default) the AgentHost A2A endpoint uses https with + /// mTLS (H1). When (PoC only) it uses plain http. Drives the + /// scheme via and is injected into the pod as + /// AgentHost__RequireMtls. Config key: Sandbox:AgentHost:RequireMtls. + /// + public bool RequireMtls { get; init; } = true; + + // ── AgentHost readiness gate (A2A cold-start race) ─────────────────────── + + /// + /// Path the AgentHost exposes for liveness/readiness on . The executor + /// polls {scheme}://{podIP}:{port}{AgentHostHealthzPath} after the claim binds and BEFORE + /// returning the A2A endpoint, so the worker never sends the first turn into the Kestrel boot + /// window (which would be refused). Default: /healthz. + /// + public string AgentHostHealthzPath { get; init; } = "/healthz"; + + /// + /// Maximum time to wait for the AgentHost to start serving + /// before failing the launch deterministically. Default: 90s (covers cold-start Kestrel bind). + /// + public int AgentHostReadyTimeoutSeconds { get; init; } = 90; + + /// Interval between AgentHost readiness probe attempts. Default: 1000ms. + public int AgentHostReadyPollIntervalMs { get; init; } = 1000; + + /// + /// Minimum age before the orphan reaper may delete an AgentHost claim that is absent from the + /// active-run map. Config key: Sandbox:Kubernetes:AgentHostClaimCreationGraceSeconds. + /// The effective value is floored above . + /// Default: 300s. + /// + public int AgentHostClaimCreationGraceSeconds { get; init; } = 300; + + /// + /// Azure Key Vault URI injected into AgentHost pods as AgentHost__KeyVaultUri so the + /// warm pod can fetch the run owner's GitHub token via workload identity at /configure-time + /// (Option C). Sourced from the API's own KV config (Auth:TokenStore:KeyVaultUri). When + /// null/empty the env var is omitted and the pod falls back to the CSI file-mount path. + /// + public string? KvUri { get; init; } +} + +/// +/// Top-level sandbox runtime options bound from the Sandbox configuration section +/// (not under Sandbox:Kubernetes). Controls the agent-execution mode and +/// the pod-release-on-suspend behaviour (Q3 hybrid). +/// +public sealed class SandboxRuntimeOptions +{ + /// + /// Agent execution mode. + /// + /// in-api (default) — run agents in-process; instant rollback path (§4.7.6). + /// pod-per-run — launch a per-run AgentHost sandbox pod; activate A2A transport. + /// + /// + public string AgentExecutionMode { get; init; } = "in-api"; + + /// + /// When true (default) and is pod-per-run, + /// the AgentHost pod is released (SandboxClaim deleted) whenever the MAF graph suspends + /// at a RequestPort (HITL/review gate) or the coordinator idles awaiting children. + /// Set to false to keep the pod warm across suspension (lower resume latency, higher + /// resource cost; recommended only for short-wait HITL in dev/staging). + /// + public bool ReleasePodOnSuspend { get; init; } = true; + + /// + public bool IsPodPerRun => + string.Equals(AgentExecutionMode, "pod-per-run", StringComparison.OrdinalIgnoreCase); +} + +/// +/// Executes sandboxed commands inside a pre-warmed Kubernetes pod obtained via a +/// SandboxClaim CRD. Lifecycle: +/// +/// Create a SandboxClaim resource (adopts a warm pod from the pool). +/// Poll until the claim transitions to phase: Bound and reports a pod name. +/// Run the command via pod-exec (Kubernetes WebSocket exec API). +/// Delete the claim on completion (controller GC cleans up the pod and service). +/// +/// Automatically selected by the API when KUBERNETES_SERVICE_HOST is present +/// (see ). +/// +internal sealed class KubernetesSandboxExecutor : ISandboxExecutor, IAgentHostPodLifecycle +{ + private const string ApiGroup = SandboxClaimConventions.ApiGroup; + private const string ApiVersion = SandboxClaimConventions.ApiVersion; + private const string ClaimPlural = SandboxClaimConventions.ClaimPlural; + private const string ContainerName = "agentweaver-sandbox"; + + /// + /// Bounded attempt count for — the total number of + /// tries (initial + retries) for a transient Kubernetes API fault (issue #230). A transient + /// connection reset (SocketException 104 → IOException → HttpRequestException) that used to fail + /// a subtask fatally is now retried with exponential backoff + jitter. + /// + private const int MaxK8sAttempts = 3; + + /// + /// Cadence for the heartbeat emitted while an + /// AgentHost SandboxClaim is still being provisioned (unbound). Must stay well under the + /// parent coordinator's Coordinator:SubtaskStallTimeoutMinutes (default 5 min) so each + /// provisioning wait window is punctuated by an event that keeps the outbound stream flowing and + /// resets the stall timer (issue #217, mirrors the #212 tool.approval_pending heartbeat cadence). + /// + internal static readonly TimeSpan SandboxProvisioningHeartbeatInterval = TimeSpan.FromSeconds(20); + + private readonly IKubernetes _client; + private readonly KubernetesSandboxOptions _options; + private readonly ILogger _logger; + private readonly IPodNameRegistry? _podRegistry; + private readonly IAgentHostTurnTokenRegistry? _turnTokenRegistry; + private readonly Security.IRunAuthorshipCapabilityStore? _authorshipCapabilityStore; + // Polls the AgentHost /healthz after bind and before returning the endpoint, closing the + // A2A cold-start race (pod Running ~20-30s before Kestrel binds :8088). Null in unit tests + // that only assert the claim body → readiness gate is skipped. + private readonly IAgentHostReadinessProbe? _readinessProbe; + // Resolves the run's submitting user so the pod can be scoped (via /configure) to the run owner's + // Copilot-entitled token instead of the installation token. Null when the run→user lookup is + // unavailable. + private readonly IRunSubmittingUserResolver? _submittingUserResolver; + // Used to POST /configure to the warm pod after bind (warm-pool deferred-config path). Null in + // unit tests → the /configure call is skipped (same null-skip convention as the readiness probe). + private readonly IHttpClientFactory? _httpClientFactory; + // Resolves the run owner's GitHub token so the API can pass it in /configure, avoiding the need + // for the kata VM pod to call Azure AD or Key Vault (blocked by Cilium FQDN policies). + private readonly IGitHubTokenStore? _tokenStore; + private readonly IGitHubTokenScopeProvider? _tokenScopeProvider; + // Refresh-aware token accessor (issue #523): a Build & Test gate can launch its AgentHost pod for + // the FIRST time (a fresh pod, not yet /configure'd for this run) many minutes after the run's + // earlier subtask stages — long enough for the submitting user's Copilot-entitled OAuth access + // token to cross its expiry skew window. Reading the raw entry via IGitHubTokenStore.GetAsync (as + // ResolveGitHubAccessTokenAsync previously did) can hand a stale/expired access token to the pod, + // which the pod then trusts unconditionally (its "fast path" skips its own Key Vault fetch + // whenever a pre-resolved token arrives) — producing GitHubCopilotUnauthorizedException at + // /configure. Routing through the same GetValidAccessTokenAsync used by GitHubCopilotClientFactory + // ensures a near-expiry token is transparently rotated before being handed to a newly-launched pod. + // Null in unit tests → falls back to the raw (non-refreshing) token store read. When present, + // it is authoritative: a null/failed refresh must never fall back to the rejected raw token. + private readonly IGitHubAccessTokenProvider? _accessTokenProvider; + // Replica-safe run secret store used to persist the per-run preview-runner credential so a + // reconcile/keepalive on either API replica can re-fetch it, and to durably DELETE it on pod + // release (spec-006 decouple-preview, BLOCKER A / RESIDUAL). Null in unit tests → no minting. + private readonly ISecretStore? _secretStore; + // Durable run-event log used to emit sandbox.provisioning_pending heartbeats into the CHILD run's + // stream while its AgentHost claim is still being scheduled by Kubernetes (unbound). Keeps the + // parent coordinator's stall timer alive during a legitimately-long Pending wait (issue #217). + // Null in unit tests → the heartbeat is skipped (same null-skip convention as the readiness probe). + private readonly IRunEventStream? _runEventStream; + // Source of the per-run AutoApproveTools flag propagated to the warm pod via /configure (bug + // #221). Null in unit tests → the flag defaults false (same null-skip convention as above). + private readonly IRunOptionsStore? _runOptions; + private readonly RunRepositoryCredentialRegistry? _repositoryCredentials; + // First-class preview lifecycle reconciler. ReleaseAgentHostPodAsync derives durable + // Previewable/PreviewActive state and applies all retention or cleanup effects before deciding + // whether to delete the claim. + private readonly Agentweaver.Api.Sandbox.Preview.ISandboxPreviewService? _previewService; + + public bool IsRealIsolation => true; + public string BackendName => "kubernetes-sandbox-claim"; + public string SelectionReason => + "Kubernetes-native sandbox via SandboxClaim warm pool (Kata VM isolation, NetworkPolicy egress restriction)."; + public bool HasNetworkWarning => false; + public string? NetworkWarningMessage => null; + + internal KubernetesSandboxExecutor( + IKubernetes client, + KubernetesSandboxOptions options, + ILogger logger, + IPodNameRegistry? podRegistry = null, + IAgentHostTurnTokenRegistry? turnTokenRegistry = null, + IAgentHostReadinessProbe? readinessProbe = null, + IRunSubmittingUserResolver? submittingUserResolver = null, + IHttpClientFactory? httpClientFactory = null, + IGitHubTokenStore? tokenStore = null, + ISecretStore? secretStore = null, + IRunEventStream? runEventStream = null, + IRunOptionsStore? runOptions = null, + RunRepositoryCredentialRegistry? repositoryCredentials = null, + IGitHubAccessTokenProvider? accessTokenProvider = null, + Agentweaver.Api.Sandbox.Preview.ISandboxPreviewService? previewService = null, + IGitHubTokenScopeProvider? tokenScopeProvider = null, + Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null) + { + _client = client; + _options = options; + _logger = logger; + _podRegistry = podRegistry; + _turnTokenRegistry = turnTokenRegistry; + _readinessProbe = readinessProbe; + _submittingUserResolver = submittingUserResolver; + _httpClientFactory = httpClientFactory; + _tokenStore = tokenStore; + _tokenScopeProvider = tokenScopeProvider; + _secretStore = secretStore; + _runEventStream = runEventStream; + _runOptions = runOptions; + _repositoryCredentials = repositoryCredentials; + _accessTokenProvider = accessTokenProvider; + _previewService = previewService; + _authorshipCapabilityStore = authorshipCapabilityStore; + } + + public async Task ExecuteAsync( + SandboxCommand command, CancellationToken ct = default) + { + // Use the Agentweaver run ID as the claim name when available so the pod can be + // looked up by run ID later (preview port-forward). Fall back to a random ID. + var claimName = string.IsNullOrEmpty(command.AgentweaverRunId) + ? $"run-{Guid.NewGuid():N}"[..20] + : SandboxClaimConventions.DeriveRunCommandClaimName(command.AgentweaverRunId); + + var requestedTimeoutMs = command.TimeoutMs > 0 + ? command.TimeoutMs + : _options.TimeoutSeconds * 1000; + var maxCommandTimeoutMs = Math.Max(1000, (_options.TimeoutSeconds * 1000) - 30_000); + var timeoutMs = Math.Min(requestedTimeoutMs, maxCommandTimeoutMs); + if (timeoutMs < requestedTimeoutMs) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: command timeout clamped from {RequestedMs}ms to {TimeoutMs}ms so it stays below SandboxClaim TTL ({TtlSeconds}s)", + requestedTimeoutMs, timeoutMs, _options.TimeoutSeconds); + } + + string podWorkingDirectory; + try + { + podWorkingDirectory = ResolvePodWorkingDirectory(command.WorkingDirectory); + } + catch (Exception ex) + { + _logger.LogError(ex, + "KubernetesSandboxExecutor: invalid workspace path {WorkingDirectory}; configured mount is {WorkspaceMountPath}", + command.WorkingDirectory, _options.WorkspaceMountPath); + return new SandboxExecResult(1, "", ex.Message, false, false); + } + + _logger.LogInformation( + "KubernetesSandboxExecutor: using workspace path {WorkspacePath} for claim {Claim} (requested {RequestedWorkingDirectory})", + podWorkingDirectory, claimName, command.WorkingDirectory); + + using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct); + linked.CancelAfter(timeoutMs); + var token = linked.Token; + var claimCreated = false; + + try + { + _logger.LogInformation( + "KubernetesSandboxExecutor: creating SandboxClaim {Claim}", claimName); + claimCreated = await CreateClaimAsync(claimName, token); + + var podName = await WaitForBoundAsync(claimName, token); + _logger.LogInformation( + "KubernetesSandboxExecutor: claim {Claim} bound to pod {Pod}", claimName, podName); + + // Register pod name so PortForwardService can locate it by Agentweaver run ID. + // Run-scoped mappings are cleared by run lifecycle cleanup, not per command, so + // preview tunnels can remain available for the whole run while the claim TTL is valid. + if (!string.IsNullOrEmpty(command.AgentweaverRunId)) + _podRegistry?.Register(command.AgentweaverRunId, podName); + + return await ExecInPodAsync(podName, command, podWorkingDirectory, token); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: timed out waiting for claim {Claim}", claimName); + return new SandboxExecResult(-1, "", "Timed out waiting for sandbox pod.", true, false); + } + finally + { + if (claimCreated && string.IsNullOrEmpty(command.AgentweaverRunId)) + await DeleteClaimAsync(claimName); + else if (claimCreated) + _logger.LogDebug( + "KubernetesSandboxExecutor: retaining SandboxClaim {Claim} for run {RunId} preview until run cleanup or TTL", + claimName, command.AgentweaverRunId); + } + } + + public async IAsyncEnumerable StreamAsync( + SandboxCommand command, + [EnumeratorCancellation] CancellationToken ct = default) + { + var result = await ExecuteAsync(command, ct); + foreach (var line in result.Stdout.Split('\n')) + yield return new SandboxOutputChunk(SandboxOutputStream.Stdout, line); + if (!string.IsNullOrEmpty(result.Stderr)) + foreach (var line in result.Stderr.Split('\n')) + yield return new SandboxOutputChunk(SandboxOutputStream.Stderr, line); + yield return new SandboxOutputChunk(SandboxOutputStream.ExitCode, result.ExitCode.ToString()); + } + + // ── IAgentHostPodLifecycle — pod-per-run lifecycle (spec §9 / Q3) ───────────── + + /// + public Task LaunchAgentHostPodAsync(string runId, CancellationToken ct = default) => + LaunchAgentHostPodAsync(runId, new AgentHostLaunchContext(SharedWorkingDirectory: null), ct); + + /// + public Task LaunchAgentHostPodAsync( + string runId, + string? workingDirectoryOverride, + CancellationToken ct = default) => + LaunchAgentHostPodAsync( + runId, + new AgentHostLaunchContext(SharedWorkingDirectory: workingDirectoryOverride), + ct); + + /// + public async Task LaunchAgentHostPodAsync( + string runId, + AgentHostLaunchContext launchContext, + CancellationToken ct = default) + { + var claimName = SandboxClaimConventions.DeriveAgentHostClaimName(runId); + var requestedWorkingDirectory = string.IsNullOrWhiteSpace(launchContext.SharedWorkingDirectory) + ? null + : Path.GetFullPath(launchContext.SharedWorkingDirectory); + + _logger.LogInformation( + "KubernetesSandboxExecutor: launching AgentHost pod for run {RunId} via claim {Claim}", + runId, claimName); + + // Resolve the run's submitting user so the pod can scope GitHub Copilot auth to that user's + // signed-in token. The user's Key Vault secret name (Option C warm-pool path) is derived here + // and delivered to the pod via /configure — never another user's secret. + var submittingUser = await ResolveSubmittingUserAsync(runId, ct).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(submittingUser)) + { + throw new InvalidOperationException( + $"Cannot launch AgentHost pod for run '{runId}' without a submitting user; " + + "the /configure call must scope the pod to the run owner's Key Vault token."); + } + + _logger.LogInformation( + "KubernetesSandboxExecutor: resolved submitting user for run {RunId}; will configure pod via /configure.", + runId); + + var (configProjectId, configAgentName) = _submittingUserResolver is not null + ? await _submittingUserResolver.GetRunIdentityAsync(runId, ct).ConfigureAwait(false) + : (null, null); + + // ghtok-user--{base32(userId)} — the SAME mapping the API uses when persisting the token to KV. + // With Entra sign-in the user's credentials live under the ACTIVE linked GitHub identity's + // scope (user-link:{oid}:{login}), so resolve the effective scope rather than assuming the + // legacy per-user scope, which is never written in that mode. + var effectiveScope = _tokenScopeProvider is not null + ? await _tokenScopeProvider + .ResolveAsync(submittingUser!, configProjectId, ct) + .ConfigureAwait(false) + : _tokenStore is IEffectiveGitHubTokenScopeResolver scopeResolver + ? await scopeResolver.ResolveEffectiveScopeAsync(submittingUser!, ct).ConfigureAwait(false) + : GitHubTokenScope.ForUser(submittingUser!); + var kvUserSecretName = KeyVaultSecretStore.SanitizeKey(effectiveScope.Key); + var turnToken = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)); + var claimCreated = false; + try + { + // Bind to the SHARED, pre-warmed AgentHost warm pool (replicas: 2). No per-run SPC, + // SandboxTemplate, or warm pool — the pod is already warm and gets its per-run context + // via the /configure POST below. + claimCreated = await CreateAgentHostClaimAsync( + claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); + + if (!claimCreated && launchContext.Purpose == AgentHostPurpose.OperatorAssistant) + { + // Every operator turn carries the CURRENT browser/platform bearer. An orphaned + // claim from a crashed prior turn is already configured with the old credential + // and /configure is intentionally one-shot, so it must never be reused. + _logger.LogInformation( + "KubernetesSandboxExecutor: recreating existing AgentHost claim {Claim} for a fresh operator-assistant caller credential.", + claimName); + await DeleteClaimAsync(claimName).ConfigureAwait(false); + _podRegistry?.Unregister(runId); + _turnTokenRegistry?.UnregisterTurnToken(runId); + await Task.Delay(1000, ct).ConfigureAwait(false); + claimCreated = await CreateAgentHostClaimAsync( + claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); + if (!claimCreated) + { + throw new InvalidOperationException( + $"AgentHost claim '{claimName}' was deleted to refresh the operator-assistant caller credential, " + + "but the replacement create still conflicted."); + } + } + else if (!claimCreated && launchContext.WorkspaceMode != ExecutionWorkspaceMode.Shared) + { + _logger.LogInformation( + "KubernetesSandboxExecutor: recreating existing AgentHost claim {Claim} for immutable pod-local workspace configuration (mode={Mode}).", + claimName, + launchContext.WorkspaceMode); + await DeleteClaimAsync(claimName).ConfigureAwait(false); + _podRegistry?.Unregister(runId); + _turnTokenRegistry?.UnregisterTurnToken(runId); + await Task.Delay(1000, ct).ConfigureAwait(false); + claimCreated = await CreateAgentHostClaimAsync( + claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); + if (!claimCreated) + { + throw new InvalidOperationException( + $"AgentHost claim '{claimName}' was deleted for immutable pod-local workspace configuration, " + + "but the replacement create still conflicted."); + } + } + else if (!claimCreated && requestedWorkingDirectory is not null) + { + var existingWorkingDirectory = await TryGetAgentHostClaimWorkingDirectoryAsync(claimName, ct) + .ConfigureAwait(false); + var sameWorktree = string.Equals( + existingWorkingDirectory, requestedWorkingDirectory, StringComparison.Ordinal); + var hasTurnToken = !string.IsNullOrWhiteSpace(_turnTokenRegistry?.TryGetTurnToken(runId)); + + if (!sameWorktree || !hasTurnToken) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: existing AgentHost claim {Claim} for run {RunId} " + + "is not reusable (sameWorktree={SameWorktree}, hasTurnToken={HasTurnToken}); recreating.", + claimName, runId, sameWorktree, hasTurnToken); + await DeleteClaimAsync(claimName).ConfigureAwait(false); + _podRegistry?.Unregister(runId); + _turnTokenRegistry?.UnregisterTurnToken(runId); + await Task.Delay(1000, ct).ConfigureAwait(false); + claimCreated = await CreateAgentHostClaimAsync( + claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); + if (!claimCreated) + { + throw new InvalidOperationException( + $"AgentHost claim '{claimName}' for run '{runId}' was deleted for worktree reconfiguration, " + + "but the replacement create still conflicted. Retrying later avoids reusing a token-less or stale pod."); + } + } + } + + var podName = await WaitForBoundWithProvisioningHeartbeatAsync(runId, claimName, ct).ConfigureAwait(false); + _logger.LogInformation( + "KubernetesSandboxExecutor: AgentHost claim {Claim} bound to pod {Pod}", claimName, podName); + + // Register also persists sandbox.execution_pod.bound into the shared RunEvents store so + // graph snapshots/deltas on any API replica can resolve the execution pod. + _podRegistry?.Register(runId, podName); + if (claimCreated) + _turnTokenRegistry?.RegisterTurnToken(runId, turnToken); + + var activeTurnToken = claimCreated + ? turnToken + : _turnTokenRegistry?.TryGetTurnToken(runId); + if (_authorshipCapabilityStore is not null && !string.IsNullOrWhiteSpace(activeTurnToken)) + { + await _authorshipCapabilityStore.RegisterAsync( + runId, activeTurnToken, DateTimeOffset.UtcNow.AddDays(1), ct).ConfigureAwait(false); + } + + var podIp = await GetPodIpAsync(podName, ct).ConfigureAwait(false); + + var endpointUrl = AgentHostEndpoint.Build( + _options.RequireMtls, podIp, _options.AgentHostPort, _options.AgentHostA2APath); + + // A2A cold-start gate: the claim binds when the pod is Running, but the AgentHost Kestrel + // listener takes ~20-30s more to bind :8088. Without this wait the worker's first A2A POST + // hits a closed port → "Connection refused" → the run fails mid-turn. Poll /healthz until the + // app is actually serving so a not-yet-ready pod is a deterministic LAUNCH failure instead. + // NOTE: a warm/standby pod serves /healthz BEFORE /configure (the readiness gate exempts + // /configure), so this confirms reachability prior to injecting the run context. + if (_readinessProbe is not null) + { + var scheme = AgentHostEndpoint.Scheme(_options.RequireMtls); + var readinessUrl = + $"{scheme}://{podIp}:{_options.AgentHostPort}{_options.AgentHostHealthzPath}"; + + _logger.LogInformation( + "KubernetesSandboxExecutor: waiting for AgentHost readiness for run {RunId} at {Url}", + runId, readinessUrl); + + try + { + await _readinessProbe.WaitUntilReadyAsync(readinessUrl, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + throw new InvalidOperationException( + $"AgentHost pod '{podName}' for run '{runId}' did not become ready at {readinessUrl} " + + $"within {_options.AgentHostReadyTimeoutSeconds}s; failing the launch.", ex); + } + } + + // Warm-pool deferred /configure: inject the per-run RunId/UserId/TurnBearerToken and the + // KV secret name into the already-warm pod, which then runs SetupAsync and becomes ready. + // Normal roles use the shared orchestration worktree. Local workspace modes carry + // immutable source refs; AgentHost creates their effective root inside execution-scratch. + if (claimCreated) + { + var repositoryAccessToken = _repositoryCredentials is null + ? null + : await _repositoryCredentials.MintAsync(runId, ct).ConfigureAwait(false); + var effectiveWorkingDirectory = await CallAgentHostConfigureAsync( + podIp, _options.AgentHostPort, runId, submittingUser, turnToken, kvUserSecretName, + effectiveScope, + await ResolveGitHubAccessTokenAsync(effectiveScope, submittingUser, ct).ConfigureAwait(false), + repositoryAccessToken, + requestedWorkingDirectory ?? await ResolveWorkingDirectoryAsync(runId, ct).ConfigureAwait(false), + launchContext, + configProjectId, + configAgentName, + ct) + .ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(effectiveWorkingDirectory)) + _podRegistry?.RegisterEffectiveWorkingDirectory(runId, effectiveWorkingDirectory); + } + else + { + _logger.LogInformation( + "KubernetesSandboxExecutor: reusing already-configured AgentHost claim {Claim} for run {RunId}", + claimName, runId); + } + + _podRegistry?.RegisterAgentEndpoint(runId, endpointUrl); + + _logger.LogInformation( + "KubernetesSandboxExecutor: AgentHost A2A endpoint for run {RunId} = {Endpoint}", + runId, endpointUrl); + + return endpointUrl; + } + catch + { + if (claimCreated) + await DeleteClaimAsync(claimName).ConfigureAwait(false); + _podRegistry?.Unregister(runId); + _turnTokenRegistry?.UnregisterTurnToken(runId); + if (_authorshipCapabilityStore is not null) + { + await _authorshipCapabilityStore.RemoveAsync(runId, CancellationToken.None) + .ConfigureAwait(false); + } + // Crash/timeout during launch: delete any credential minted before the failure so it is + // never left behind (spec-006 decouple-preview, RESIDUAL rev3 gap). + await DeletePreviewRunnerCredentialAsync(runId, CancellationToken.None).ConfigureAwait(false); + await RevokeRepositoryCredentialAsync(runId, CancellationToken.None).ConfigureAwait(false); + throw; + } + } + + /// + public async Task ReleaseAgentHostPodAsync(string runId, CancellationToken ct = default) + { + var claimName = SandboxClaimConventions.DeriveAgentHostClaimName(runId); + + // Issue #542: if a live preview is still active for this run, releasing the pod here (at the + // originating subtask's turn end) would 404 the preview URL before any human-review gate or + // demo viewer can open it. Defer the claim delete while the preview is alive; the preview's own + // idle/max expiry + the reaper will eventually reap the pod, so this cannot leak. + if (_previewService is not null && + await _previewService.ReconcilePreviewLifecycleAsync(runId, ct).ConfigureAwait(false) + == Agentweaver.Api.Sandbox.Preview.PreviewLifecycleState.PreviewActive) + { + _logger.LogInformation( + "KubernetesSandboxExecutor: deferring AgentHost pod release for run {RunId} (claim " + + "{Claim}) — a live preview is still active; the preview idle/max expiry will reap it.", + runId, claimName); + return; + } + + _logger.LogInformation( + "KubernetesSandboxExecutor: releasing AgentHost pod for run {RunId} (claim {Claim})", + runId, claimName); + + await DeleteClaimAsync(claimName, ct).ConfigureAwait(false); + _podRegistry?.Unregister(runId); + _turnTokenRegistry?.UnregisterTurnToken(runId); + if (_authorshipCapabilityStore is not null) + await _authorshipCapabilityStore.RemoveAsync(runId, ct).ConfigureAwait(false); + await DeletePreviewRunnerCredentialAsync(runId, ct).ConfigureAwait(false); + await RevokeRepositoryCredentialAsync(runId, ct).ConfigureAwait(false); + + _logger.LogInformation( + "KubernetesSandboxExecutor: AgentHost pod released for run {RunId}", runId); + } + + /// + /// Resolves the submitting user for via the injected resolver, never + /// throwing (a lookup failure must not fail the launch — it degrades to omitting the user id). + /// + private async Task ResolveSubmittingUserAsync(string runId, CancellationToken ct) + { + if (_submittingUserResolver is null) + return null; + + try + { + return await _submittingUserResolver.GetSubmittingUserAsync(runId, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "KubernetesSandboxExecutor: failed to resolve submitting user for run {RunId}; " + + "AgentHost__UserId will be omitted.", + runId); + return null; + } + } + + /// + /// Resolves the per-run working directory (shared orchestration worktree path) for + /// via the injected resolver, never throwing (a lookup failure must not + /// fail the launch — it degrades to omitting the working directory, so the pod falls back to its + /// static AgentHost__WorkingDirectory env default). + /// + private async Task ResolveWorkingDirectoryAsync(string runId, CancellationToken ct) + { + if (_submittingUserResolver is null) + return null; + + try + { + return await _submittingUserResolver.GetWorkingDirectoryAsync(runId, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "KubernetesSandboxExecutor: failed to resolve working directory for run {RunId}; " + + "AgentHost__WorkingDirectory env default will be used.", + runId); + return null; + } + } + /// (AgentHostWarmPoolRef, replicas: 2). No spec.env is injected — the v0.5.0 + /// controller bypasses warm pool adoption whenever spec.env or + /// spec.volumeClaimTemplates are present. All static config lives in the SandboxTemplate + /// or agenthost-config ConfigMap. The per-run context (RunId / UserId / TurnBearerToken / + /// KV secret name) is delivered after bind via POST /configure + /// (). + /// + private async Task CreateAgentHostClaimAsync( + string claimName, string warmPoolName, string? workingDirectory, string runId, CancellationToken ct) + { + var annotations = new Dictionary + { + // Persist the ORIGINAL run id so the reaper can recover it from an orphaned claim (the + // claim name is a lossy 12-char derivation) and delete run-scoped side artifacts such as + // the per-run preview-runner credential (spec-006 decouple-preview). + [SandboxClaimConventions.RunIdAnnotation] = runId, + }; + if (!string.IsNullOrWhiteSpace(workingDirectory)) + annotations["agentweaver.io/working-directory"] = workingDirectory; + + var manifest = new + { + apiVersion = $"{ApiGroup}/{ApiVersion}", + kind = "SandboxClaim", + metadata = new + { + name = claimName, + @namespace = _options.Namespace, + annotations = annotations.Count == 0 ? null : annotations, + }, + spec = new + { + // v0.5.0 v1beta1 SandboxClaimSpec: spec.warmPoolRef.name references the + // SandboxWarmPool to bind from. sandboxTemplateRef+warmpool were the + // v0.4.x/v1alpha1 deprecated fields. + warmPoolRef = new { name = warmPoolName }, + lifecycle = new { ttlSecondsAfterFinished = _options.TimeoutSeconds, shutdownPolicy = "Delete" }, + }, + }; + + // Idempotent create with bounded transient-fault retry (issue #230). A mid-flight connection + // reset can commit the SandboxClaim server-side BEFORE we observe the response, so the retry + // may see a 409 for OUR OWN create — handled attempt-awarely below. + for (var attempt = 1; ; attempt++) + { + try + { + await _client.CustomObjects.CreateNamespacedCustomObjectAsync( + manifest, ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, + cancellationToken: ct).ConfigureAwait(false); + return true; + } + catch (HttpOperationException ex) when (ex.Response?.StatusCode == System.Net.HttpStatusCode.Conflict) + { + if (attempt > 1) + { + // Retry-409: a transient reset committed our create server-side before we saw the + // response, and this retry now observes our own claim. We own it → return true so + // the caller registers the turn token and runs /configure exactly as on a 200, + // rather than taking the silent "reuse pre-existing claim" path (which would leave + // the pod un-configured and token-less). + _logger.LogInformation( + "KubernetesSandboxExecutor: SandboxClaim {Claim} returned 409 on retry attempt {Attempt}; " + + "treating as our own create that committed before a transient reset — configuring it.", + claimName, attempt); + return true; + } + + // First-attempt 409: a genuinely pre-existing claim owned by an earlier launch. + _logger.LogInformation( + "KubernetesSandboxExecutor: SandboxClaim {Claim} already exists; waiting for existing claim", + claimName); + return false; + } + catch (Exception ex) when (attempt < MaxK8sAttempts && IsTransientK8sFault(ex, ct)) + { + var delay = BackoffWithJitter(attempt); + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: transient fault creating SandboxClaim {Claim} on attempt " + + "{Attempt}/{Max}; retrying in {DelayMs}ms.", + claimName, attempt, MaxK8sAttempts, (int)delay.TotalMilliseconds); + await Task.Delay(delay, ct).ConfigureAwait(false); + } + } + } + + // ── Transient Kubernetes API resilience (issue #230) ────────────────────────── + + /// + /// Executes an idempotent Kubernetes API call with a bounded retry ( + /// total attempts) over transient faults only — a mid-flight connection reset + /// (SocketException 104 → IOException → HttpRequestException), a 429/5xx from the API server, or an + /// HttpClient timeout. Caller cancellation is never retried and aborts the backoff immediately + /// (await Task.Delay(delay, ct)). Non-transient faults (e.g. 404/409/422) propagate on the + /// first attempt. MUST NOT wrap non-idempotent calls (e.g. the AgentHost POST /configure, + /// whose second delivery 409-hard-fails). + /// + private async Task ExecuteK8sWithRetryAsync( + Func> operation, CancellationToken ct) + { + for (var attempt = 1; ; attempt++) + { + try + { + return await operation(ct).ConfigureAwait(false); + } + catch (Exception ex) when (attempt < MaxK8sAttempts && IsTransientK8sFault(ex, ct)) + { + var delay = BackoffWithJitter(attempt); + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: transient Kubernetes API fault on attempt {Attempt}/{Max}; " + + "retrying in {DelayMs}ms.", attempt, MaxK8sAttempts, (int)delay.TotalMilliseconds); + await Task.Delay(delay, ct).ConfigureAwait(false); + } + } + } + + /// + /// Exponential backoff (~250ms · 2^(attempt-1), capped at ~2s) plus 0-250ms jitter to de-sync + /// concurrent launches retrying against the same API server after a blip. + /// + private static TimeSpan BackoffWithJitter(int attempt) + { + var baseMs = Math.Min(250 * (1 << (attempt - 1)), 2000); + var jitterMs = Random.Shared.Next(0, 250); + return TimeSpan.FromMilliseconds(baseMs + jitterMs); + } + + /// + /// True only for faults worth retrying an idempotent k8s call over: 429/5xx from the API server, + /// a socket/IO connection reset (directly or nested in an inner exception), or an HttpClient + /// timeout ( with no caller cancellation). Caller + /// cancellation short-circuits to false so a genuine cancel is never retried. A 409 Conflict is + /// intentionally NOT transient here — it is handled separately (idempotent create semantics). + /// + private static bool IsTransientK8sFault(Exception ex, CancellationToken ct) + { + if (ct.IsCancellationRequested) return false; // caller cancel — never retry + switch (ex) + { + case HttpOperationException k when k.Response is not null: + var s = (int)k.Response.StatusCode; + return s == 429 || s >= 500; // 409 handled separately, NOT here + case HttpRequestException: return true; + case IOException: return true; + case OperationCanceledException: // includes TaskCanceledException (HttpClient timeout) + return !ct.IsCancellationRequested; + } + for (Exception? i = ex.InnerException; i is not null; i = i.InnerException) + if (i is SocketException or IOException) return true; + return false; + } + + private async Task TryGetAgentHostClaimWorkingDirectoryAsync(string claimName, CancellationToken ct) + { + try + { + var raw = await _client.CustomObjects.GetNamespacedCustomObjectAsync( + ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, + cancellationToken: ct).ConfigureAwait(false); + var json = JsonSerializer.Serialize(raw); + using var doc = JsonDocument.Parse(json); + if (doc.RootElement.TryGetProperty("metadata", out var meta) && + meta.TryGetProperty("annotations", out var ann) && + ann.TryGetProperty("agentweaver.io/working-directory", out var wd) && + wd.ValueKind == JsonValueKind.String) + return wd.GetString(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to read working-directory annotation for claim {Claim}", + claimName); + } + + return null; + } + + /// + /// Resolves the run owner's GitHub access token from the API-side token store so it can be + /// forwarded in the /configure body. The kata VM pod cannot reach Azure AD or Key Vault + /// (Cilium FQDN policies use eBPF interception that doesn't cross the guest kernel boundary). + /// Never throws — a lookup failure degrades gracefully: the pod will attempt the KV fetch itself + /// (which may fail) rather than causing a hard launch failure here. + /// + private async Task ResolveGitHubAccessTokenAsync( + GitHubTokenScope scope, + string userId, + CancellationToken ct) + { + // Prefer the refresh-aware provider (issue #523): a fresh AgentHost pod launched late in a + // long-running assembly (e.g. the Build & Test gate, well after the run's earlier subtask + // stages) can be handed a near-expiry or already-expired access token if we only ever read + // the raw stored entry — the pod's "fast path" trusts a pre-resolved token unconditionally + // and never re-validates it against Key Vault or GitHub. Routing through + // GetValidAccessTokenAsync mirrors GitHubCopilotClientFactory.CreateClientAsync and + // transparently rotates the token before it is handed to the pod. + if (_accessTokenProvider is not null) + { + try + { + var refreshed = await _accessTokenProvider.GetValidAccessTokenAsync(scope, ct) + .ConfigureAwait(false); + if (string.IsNullOrEmpty(refreshed)) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: refresh-aware GitHub token provider returned no valid credential " + + "for {UserId} (scope {Scope}); refusing raw-token fallback.", + userId, + scope.Key); + } + return refreshed; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to resolve/refresh GitHub token for {UserId} via " + + "IGitHubAccessTokenProvider (scope {Scope}); refusing raw-token fallback.", + userId, + scope.Key); + return null; + } + } + + if (_tokenStore is null) + return null; + + try + { + var entry = await _tokenStore.GetAsync(scope, ct).ConfigureAwait(false); + if (entry.Status == GitHubTokenStatus.SignedIn && !string.IsNullOrEmpty(entry.AccessToken)) + return entry.AccessToken; + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to pre-resolve GitHub token for {UserId} — pod will fall back to KV.", + userId); + } + + return null; + } + + /// + /// Injects the per-run context into an already-warm AgentHost pod via its one-time + /// POST /configure endpoint. The pod then fetches ONLY + /// from Key Vault (its configured user's token) and runs SetupAsync. The endpoint is guarded by + /// NetworkPolicy (ingress to AgentHost pods restricted to API/worker), not the TurnBearerToken + /// (which is itself delivered here). Idempotency: a second call returns 409 and is treated as a + /// hard launch failure. + /// + private async Task CallAgentHostConfigureAsync( + string podIp, int port, string runId, string userId, string turnBearerToken, + string kvUserSecretName, GitHubTokenScope tokenScope, string? gitHubAccessToken, + string? repositoryAccessToken, + string? sharedWorkingDirectory, + AgentHostLaunchContext launchContext, + string? projectId, + string? agentName, + CancellationToken ct) + { + if (_httpClientFactory is null) + { + // No HttpClient available (unit tests). Mirrors the readiness-probe null-skip; in-cluster + // the factory is always present, so this never short-circuits a real launch. + _logger.LogWarning( + "KubernetesSandboxExecutor: no IHttpClientFactory — skipping /configure for run {RunId}.", + runId); + return null; + } + + var scheme = AgentHostEndpoint.Scheme(_options.RequireMtls); + var configureUrl = $"{scheme}://{podIp}:{port}/configure"; + + // Mint a FRESH per-run preview-runner credential (spec-006 decouple-preview, BLOCKER A). + // Delivered in-memory via this /configure body ONLY (never pod env/file), and persisted to the + // run secret store so any replica can re-fetch it for reconcile/keepalive. Durably deleted on + // pod release. Every launch/relaunch mints a new value — the old one is never reused. + var previewRunnerCredential = await MintPreviewRunnerCredentialAsync(runId, ct).ConfigureAwait(false); + + var body = new + { + runId, + userId, + turnBearerToken, + kvUserSecretName, + gitHubAccessToken, + repositoryAccessToken, + callerBearerToken = launchContext.CallerBearerToken, + // Keep the legacy property during rolling upgrades; new AgentHosts prefer the explicit + // sharedWorkingDirectory descriptor and create any local workspace inside the pod. + workingDirectory = sharedWorkingDirectory, + sharedWorkingDirectory, + previewRunnerCredential, + purpose = launchContext.Purpose.ToString(), + launchContext.SourceRepositoryPath, + launchContext.SourceRef, + launchContext.BaseCommitSha, + launchContext.ExpectedTreeHash, + workspaceMode = launchContext.WorkspaceMode.ToString(), + launchContext.ScratchRoot, + launchContext.CommitAuthorName, + launchContext.CommitAuthorEmail, + // Per-run AutoApproveTools flag (bug #221). Resolved from the API-side run-options store + // keyed by the child runId; defaults false when the store is unavailable (unit tests). + autoApproveTools = _runOptions?.Get(runId).AutoApproveTools ?? false, + // Per-run project/agent identity (#335). Delivered so the in-pod agent's tool schema + // includes the Agentweaver API tools (record_memory, get_memory, submit_decision, + // list_decisions, list_inbox). Warm pods boot with an empty static AgentHost__ProjectId + // /AgentName, so without these the memory/decision tools never reach the agent. + projectId, + agentName, + }; + + _logger.LogInformation( + "KubernetesSandboxExecutor: configuring AgentHost pod for run {RunId} at {Url}", + runId, configureUrl); + + using var client = _httpClientFactory.CreateClient(HttpAgentHostReadinessProbe.HttpClientName); + using var response = await client + .PostAsJsonAsync(configureUrl, body, ct) + .ConfigureAwait(false); + var detail = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + var reason = "agenthost_configure_failed"; + try + { + using var document = JsonDocument.Parse(detail); + if (document.RootElement.TryGetProperty("error", out var error) + && error.ValueKind == JsonValueKind.String + && !string.IsNullOrWhiteSpace(error.GetString())) + reason = error.GetString()!; + } + catch (JsonException) + { + // Plain-text legacy errors keep the generic typed reason. + } + + if (string.Equals( + reason, + "agenthost_configure_copilot_unauthorized", + StringComparison.Ordinal) && + _accessTokenProvider is not null) + { + var refreshed = await _accessTokenProvider + .RefreshAfterUnauthorizedAsync(tokenScope, gitHubAccessToken, ct) + .ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(refreshed) && + !string.Equals(refreshed, gitHubAccessToken, StringComparison.Ordinal)) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: AgentHost /configure rejected the Copilot credential for run {RunId}; " + + "scope {Scope} was refreshed and the pod must be recreated (recoveryAttempt=1, maxRecoveryAttempts=1).", + runId, + tokenScope.Key); + throw new AgentHostConfigureException( + "agenthost_configure_copilot_token_refreshed", + $"AgentHost /configure rejected the Copilot credential for run '{runId}'. " + + "The credential was refreshed; recreate the one-time-configured pod and retry once.", + (int)response.StatusCode, + retryable: true, + recoveryAction: "recreate_pod_with_refreshed_credential"); + } + + _logger.LogWarning( + "KubernetesSandboxExecutor: AgentHost /configure rejected the Copilot credential for run {RunId}; " + + "scope {Scope} could not produce a different refreshed credential, so the failure is not retryable.", + runId, + tokenScope.Key); + } + + throw new AgentHostConfigureException( + reason, + $"AgentHost /configure for run '{runId}' failed: HTTP {(int)response.StatusCode} {detail}", + (int)response.StatusCode); + } + + if (string.IsNullOrWhiteSpace(detail)) + return null; + + try + { + using var document = JsonDocument.Parse(detail); + if (document.RootElement.TryGetProperty("effectiveWorkingDirectory", out var path) + && path.ValueKind == JsonValueKind.String + && !string.IsNullOrWhiteSpace(path.GetString())) + { + return path.GetString(); + } + } + catch (JsonException ex) + { + _logger.LogWarning( + ex, + "KubernetesSandboxExecutor: AgentHost /configure for run {RunId} returned an invalid success body; preview will use the shared working directory.", + runId); + } + + return null; + } + + /// + /// Mints and persists a fresh per-run preview-runner credential and returns it for in-memory + /// delivery via /configure. Returns when no secret store is + /// available (unit tests) — the pod then relies on the turn token only. The persisted key is + /// derived deterministically from the run id () + /// so the release-time delete matches (spec-006 decouple-preview, BLOCKER A). + /// + private async Task MintPreviewRunnerCredentialAsync(string runId, CancellationToken ct) + { + if (_secretStore is null) + return string.Empty; + + var credential = Preview.PreviewRunnerCredential.Mint(); + var key = Preview.PreviewRunnerCredential.SecretKey(runId); + try + { + await _secretStore.SetSecretAsync(key, credential, etag: null, ct).ConfigureAwait(false); + _logger.LogInformation( + "KubernetesSandboxExecutor: minted per-run preview-runner credential for run {RunId}", runId); + return credential; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // Best-effort: a persist failure must not fail the launch. The pod still receives the + // credential in-memory (same-process affinity uses the turn token anyway), but a + // cross-replica reconcile could not re-fetch it — acceptable degradation. + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to persist preview-runner credential for run {RunId}; " + + "delivering in-memory only.", runId); + return credential; + } + } + + /// + /// Durably deletes the per-run preview-runner credential from the run secret store. No-op when + /// absent ( ignores a missing key). Never throws — + /// a delete failure must not break terminal cleanup. Called on EVERY terminal path (happy + /// release + crash/timeout/failed-run via the pod-release seam) so the credential's durable + /// lifetime is bounded by the pod's (spec-006 decouple-preview, RESIDUAL rev3 gap). + /// + private async Task DeletePreviewRunnerCredentialAsync(string runId, CancellationToken ct) + { + if (_secretStore is null) + return; + + try + { + await _secretStore.DeleteSecretAsync(Preview.PreviewRunnerCredential.SecretKey(runId), ct) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to delete preview-runner credential for run {RunId} (best-effort)", + runId); + } + } + + private async Task RevokeRepositoryCredentialAsync(string runId, CancellationToken ct) + { + if (_repositoryCredentials is null) + return; + + try + { + await _repositoryCredentials.RevokeAsync(runId, ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning( + ex, + "KubernetesSandboxExecutor: failed to revoke repository credential for run {RunId}", + runId); + } + } + + /// + /// Waits for the AgentHost SandboxClaim to bind while emitting periodic + /// heartbeats into the CHILD run's event + /// stream. Scheduling is Kubernetes' job: a claim may sit unbound (pod Pending) for a while until + /// a node frees up or the pool autoscales — that is FINE and must not fail the run (issue #217). + /// The heartbeat keeps the parent coordinator's subtask-stall timer alive during that legitimate + /// wait, mirroring the #212 tool.approval_pending heartbeat. Best-effort: if no + /// is wired (unit tests) this degrades to a plain + /// . + /// + private async Task WaitForBoundWithProvisioningHeartbeatAsync( + string runId, string claimName, CancellationToken ct) + { + if (_runEventStream is null) + return await WaitForBoundAsync(claimName, ct).ConfigureAwait(false); + + var boundTask = WaitForBoundAsync(claimName, ct); + while (true) + { + var delayTask = Task.Delay(SandboxProvisioningHeartbeatInterval, ct); + var completed = await Task.WhenAny(boundTask, delayTask).ConfigureAwait(false); + if (ReferenceEquals(completed, boundTask)) + return await boundTask.ConfigureAwait(false); // propagates the bound pod name / any error + + // The claim is still unbound after the heartbeat interval — emit a non-terminal + // heartbeat so the coordinator's stall window resets while Kubernetes schedules the pod. + await delayTask.ConfigureAwait(false); // observe cancellation + await EmitProvisioningPendingAsync(runId, claimName, ct).ConfigureAwait(false); + } + } + + /// + /// Appends a single heartbeat to + /// 's durable event stream. Best-effort: a stream-append failure is + /// logged and swallowed so it can never fail a launch that Kubernetes would otherwise admit. + /// + private async Task EmitProvisioningPendingAsync(string runId, string claimName, CancellationToken ct) + { + try + { + await _runEventStream!.AppendAsync(runId, new RunEvent(0, EventTypes.SandboxProvisioningPending, new + { + claimName, + timestamp_utc = DateTimeOffset.UtcNow.ToString("O"), + }), ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to emit sandbox.provisioning_pending heartbeat for run {RunId} (best-effort)", + runId); + } + } + + /// + /// Parses a Kubernetes CPU quantity into whole cores. Handles plain cores ("24", + /// "1.5") and the millicore suffix ("500m" = 0.5 cores). Returns + /// for an unrecognized format. + /// + internal static bool TryParseCpu(string? value, out double cores) + { + cores = 0; + if (string.IsNullOrWhiteSpace(value)) + return false; + + value = value.Trim(); + if (value.EndsWith("m", StringComparison.Ordinal)) + { + var millis = value[..^1]; + if (double.TryParse(millis, System.Globalization.NumberStyles.Float, + System.Globalization.CultureInfo.InvariantCulture, out var m)) + { + cores = m / 1000.0; + return true; + } + return false; + } + + return double.TryParse(value, System.Globalization.NumberStyles.Float, + System.Globalization.CultureInfo.InvariantCulture, out cores); + } + + /// + /// Reads the pod IP from the Kubernetes API after the claim is Bound. + /// Polls every 2 s until status.podIP is non-empty (pod has been scheduled + /// and assigned a network address). + /// + private async Task GetPodIpAsync(string podName, CancellationToken ct) + { + while (true) + { + ct.ThrowIfCancellationRequested(); + + var pod = await ExecuteK8sWithRetryAsync( + token => _client.CoreV1.ReadNamespacedPodAsync( + podName, _options.Namespace, cancellationToken: token), + ct).ConfigureAwait(false); + + var ip = pod?.Status?.PodIP; + if (!string.IsNullOrWhiteSpace(ip)) + return ip; + + _logger.LogDebug( + "KubernetesSandboxExecutor: waiting for pod IP of {Pod} (current: {Ip})", + podName, ip ?? "(none)"); + + await Task.Delay(2000, ct).ConfigureAwait(false); + } + } + + // ── Claim management ────────────────────────────────────────────────────────── + + private async Task CreateClaimAsync(string claimName, CancellationToken ct) + { + // The cluster service CIDR must be present in SandboxEgressCidrExclusions so + // sandbox NetworkPolicy does not accidentally allow in-cluster service egress. + var manifest = new + { + apiVersion = $"{ApiGroup}/{ApiVersion}", + kind = "SandboxClaim", + metadata = new { name = claimName, @namespace = _options.Namespace }, + spec = new + { + // v0.5.0 v1beta1 SandboxClaimSpec: spec.warmPoolRef.name references the + // SandboxWarmPool to bind from. sandboxTemplateRef+warmpool were the + // v0.4.x/v1alpha1 deprecated fields. + warmPoolRef = new { name = _options.WarmPoolRef }, + lifecycle = new { ttlSecondsAfterFinished = _options.TimeoutSeconds, shutdownPolicy = "Delete" }, + }, + }; + + try + { + await _client.CustomObjects.CreateNamespacedCustomObjectAsync( + manifest, ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, + cancellationToken: ct).ConfigureAwait(false); + return true; + } + catch (HttpOperationException ex) when (ex.Response?.StatusCode == System.Net.HttpStatusCode.Conflict) + { + _logger.LogInformation( + "KubernetesSandboxExecutor: SandboxClaim {Claim} already exists; waiting for existing claim", + claimName); + return false; + } + } + + /// + /// Polls every 2 s until the claim's Ready condition is True; returns the bound + /// pod name from status.sandbox.name. + /// + private async Task WaitForBoundAsync(string claimName, CancellationToken ct) + { + while (true) + { + ct.ThrowIfCancellationRequested(); + + var raw = await ExecuteK8sWithRetryAsync( + token => _client.CustomObjects.GetNamespacedCustomObjectAsync( + ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, + cancellationToken: token), + ct).ConfigureAwait(false); + + var json = JsonSerializer.Serialize(raw); + using var doc = JsonDocument.Parse(json); + + // Surface a controller reconcile failure (e.g. "exceeded quota") as a deterministic + // launch failure with a precise reason instead of polling until the caller times out. + var reconcilerError = SandboxClaimConventions.TryGetReconcilerError(doc.RootElement); + if (reconcilerError is not null) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: claim {Claim} reconcile failed: {Error}", + claimName, reconcilerError); + throw new AgentHostPodReconcilerErrorException( + $"SandboxClaim '{claimName}' could not be provisioned: {reconcilerError}"); + } + + var podName = SandboxClaimConventions.TryGetBoundPodName(doc.RootElement); + if (!string.IsNullOrEmpty(podName)) + return podName; + + await Task.Delay(2000, ct); + } + } + + private async Task DeleteClaimAsync(string claimName, CancellationToken ct = default) + { + try + { + await _client.CustomObjects.DeleteNamespacedCustomObjectAsync( + ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, cancellationToken: ct); + _logger.LogInformation( + "KubernetesSandboxExecutor: deleted claim {Claim}", claimName); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: could not delete claim {Claim} (best-effort)", claimName); + } + } + + // ── Command execution ───────────────────────────────────────────────────────── + + private async Task ExecInPodAsync( + string podName, SandboxCommand command, string podWorkingDirectory, CancellationToken ct) + { + const int maxOutputBytes = 4 * 1024 * 1024; + + var shellScript = BuildShellScript(command, podWorkingDirectory); + + var ws = await _client.WebSocketNamespacedPodExecAsync( + podName, _options.Namespace, + new[] { "/bin/sh", "-c", shellScript }, + container: ContainerName, + stdin: false, stdout: true, stderr: true, tty: false, + cancellationToken: ct); + + using var demux = new StreamDemuxer(ws, StreamType.RemoteCommand); + demux.Start(); + + using var stdoutStream = demux.GetStream(ChannelIndex.StdOut, null); + using var stderrStream = demux.GetStream(ChannelIndex.StdErr, null); + // Channel 3 (Error) carries the terminal v1.Status payload with the real exit code. + using var statusStream = demux.GetStream(ChannelIndex.Error, null); + + var stdoutTask = ReadBoundedAsync(stdoutStream, maxOutputBytes, ct); + var stderrTask = ReadBoundedAsync(stderrStream, maxOutputBytes, ct); + var statusTask = ReadBoundedAsync(statusStream, maxOutputBytes, ct); + + await Task.WhenAll(stdoutTask, stderrTask, statusTask); + + var (stdoutBytes, stdoutTruncated) = await stdoutTask; + var (stderrBytes, stderrTruncated) = await stderrTask; + var (statusBytes, _) = await statusTask; + + var stdout = SandboxOutputRedactor.Default.Redact(Encoding.UTF8.GetString(stdoutBytes)); + var stderr = SandboxOutputRedactor.Default.Redact(Encoding.UTF8.GetString(stderrBytes)); + var exitCode = ParseExitCode(Encoding.UTF8.GetString(statusBytes)); + + return new SandboxExecResult( + exitCode, stdout, stderr, false, stdoutTruncated || stderrTruncated); + } + + /// + /// Reads up to from a stream, stopping at the cap. + /// Returns the bytes collected and whether the output was truncated. + /// + private static async Task<(byte[] Bytes, bool Truncated)> ReadBoundedAsync( + Stream stream, int maxBytes, CancellationToken ct) + { + using var buffer = new MemoryStream(); + var chunk = new byte[8192]; + bool truncated = false; + int read; + while ((read = await stream.ReadAsync(chunk, ct)) > 0) + { + int remaining = maxBytes - (int)buffer.Length; + if (remaining <= 0) { truncated = true; break; } + int take = Math.Min(read, remaining); + buffer.Write(chunk, 0, take); + if (take < read) { truncated = true; break; } + } + return (buffer.ToArray(), truncated); + } + + /// + /// Parses the terminal v1.Status JSON emitted on channel 3. + /// status: "Success" → exit 0. status: "Failure" → the ExitCode + /// cause from details.causes (defaulting to 1 if not present). + /// + private static int ParseExitCode(string statusJson) + { + if (string.IsNullOrWhiteSpace(statusJson)) + return 0; + + try + { + using var doc = JsonDocument.Parse(statusJson); + var root = doc.RootElement; + + var status = root.TryGetProperty("status", out var s) ? s.GetString() : null; + if (string.Equals(status, "Success", StringComparison.OrdinalIgnoreCase)) + return 0; + + if (root.TryGetProperty("details", out var details) && + details.TryGetProperty("causes", out var causes) && + causes.ValueKind == JsonValueKind.Array) + { + foreach (var cause in causes.EnumerateArray()) + { + var reason = cause.TryGetProperty("reason", out var r) ? r.GetString() : null; + if (string.Equals(reason, "ExitCode", StringComparison.OrdinalIgnoreCase) && + cause.TryGetProperty("message", out var m) && + int.TryParse(m.GetString(), out var code)) + return code; + } + } + + // Failure status with no parseable ExitCode cause → non-zero. + return 1; + } + catch (JsonException) + { + return 0; + } + } + + private string ResolvePodWorkingDirectory(string requestedWorkingDirectory) + { + var mountPath = NormalizeUnixPath(_options.WorkspaceMountPath, forceAbsolute: true); + if (string.IsNullOrWhiteSpace(requestedWorkingDirectory)) + return mountPath; + + var requested = NormalizeUnixPath(requestedWorkingDirectory, forceAbsolute: false); + if (IsSameOrChildPath(requested, mountPath)) + return requested; + + throw new InvalidOperationException( + $"Kubernetes sandbox working directory '{requestedWorkingDirectory}' is not under mounted workspace '{mountPath}'. " + + "Configure Workspace:PersistentVolume:MountRoot/Workspace:Path to match the workspace PVC mount used by sandbox pods."); + } + + private static bool IsSameOrChildPath(string path, string root) => + string.Equals(path, root, StringComparison.Ordinal) + || (root == "/" && path.StartsWith("/", StringComparison.Ordinal)) + || path.StartsWith(root + "/", StringComparison.Ordinal); + + private static string NormalizeUnixPath(string path, bool forceAbsolute) + { + var normalized = path.Trim().Replace('\\', '/'); + while (normalized.Contains("//", StringComparison.Ordinal)) + normalized = normalized.Replace("//", "/", StringComparison.Ordinal); + if (forceAbsolute && !normalized.StartsWith("/", StringComparison.Ordinal)) + normalized = "/" + normalized; + return normalized.Length > 1 ? normalized.TrimEnd('/') : normalized; + } + + private static string BuildShellScript(SandboxCommand command, string podWorkingDirectory) + { + var sb = new StringBuilder(); + + if (command.Environment is { Count: > 0 }) + { + foreach (var (key, value) in command.Environment) + sb.AppendLine($"export {key}={ShellSingleQuote(value)}"); + } + + sb.AppendLine($"cd {ShellSingleQuote(podWorkingDirectory)}"); + + sb.Append(command.CommandLine); + return sb.ToString(); + } + + private static string ShellSingleQuote(string s) => + "'" + s.Replace("'", "'\\''") + "'"; +} diff --git a/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs new file mode 100644 index 000000000..1b86bbdcf --- /dev/null +++ b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs @@ -0,0 +1,93 @@ +using System.Collections.Concurrent; +using Agentweaver.Api.Auth; +using Agentweaver.Api.Memory; +using Agentweaver.Api.Webhooks; +using Agentweaver.Domain; +using Microsoft.Extensions.DependencyInjection; + +namespace Agentweaver.Api.Sandbox; + +/// +/// Holds minted repository credentials in API memory until the owning run releases its pod. +/// The registry has no command inputs and does not persist credentials. +/// +public sealed class RunRepositoryCredentialRegistry(IServiceScopeFactory scopeFactory) +{ + private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _mintLocks = new(StringComparer.Ordinal); + + public async Task MintAsync(string runId, CancellationToken ct = default) + { + var mintLock = _mintLocks.GetOrAdd(runId, static _ => new SemaphoreSlim(1, 1)); + await mintLock.WaitAsync(ct).ConfigureAwait(false); + try + { + if (_entries.TryGetValue(runId, out var current)) + { + if (current.ExpiresAt > DateTimeOffset.UtcNow) + return null; + _entries.TryRemove(runId, out _); + } + + using var scope = scopeFactory.CreateScope(); + var persistence = scope.ServiceProvider.GetRequiredService(); + var snapshot = (await persistence.GetCapabilitySnapshotsAsync(runId, ct).ConfigureAwait(false)) + .SingleOrDefault(x => x.Purpose == GitHubCapabilityPurpose.UnattendedRepository); + if (snapshot is null) + return null; + + Entry? minted = null; + var outcome = await scope.ServiceProvider.GetRequiredService() + .TryUseRepositoryCredentialAsync( + new SnapshotRef(snapshot.SnapshotRef), + DateTimeOffset.UtcNow, + (token, expiresAt) => + { + minted = new Entry(token, expiresAt); + return Task.CompletedTask; + }, + ct).ConfigureAwait(false); + if (outcome != GitHubCapabilityBrokerOutcome.Issued || minted is null) + return null; + + _entries[runId] = minted; + return minted.AccessToken; + } + finally + { + mintLock.Release(); + } + } + + public async Task RevokeAsync(string? runId, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(runId)) + return; + + var mintLock = _mintLocks.GetOrAdd(runId, static _ => new SemaphoreSlim(1, 1)); + await mintLock.WaitAsync(ct).ConfigureAwait(false); + try + { + if (!_entries.TryRemove(runId, out var entry)) + return; + + using var scope = scopeFactory.CreateScope(); + await scope.ServiceProvider.GetRequiredService() + .RevokeRepositoryTokenAsync(entry.AccessToken, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch + { + // Token expiry bounds a failed best-effort revoke. + } + finally + { + mintLock.Release(); + } + } + + private sealed record Entry(string AccessToken, DateTimeOffset ExpiresAt); +} diff --git a/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs b/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs index cb17aa3ec..51265fe58 100644 --- a/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs +++ b/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs @@ -1,181 +1,184 @@ -using k8s; -using Agentweaver.SandboxExec; -using Agentweaver.AgentRuntime.Workflow; -using Agentweaver.Api.Infrastructure; -using Agentweaver.Domain; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; - -namespace Agentweaver.Api.Sandbox; - -/// -/// Selects ISandboxExecutor based on: -/// 1. Sandbox:Backend config override ("kubernetes" or "local"). -/// 2. KUBERNETES_SERVICE_HOST environment variable (implicit in-cluster probe). -/// -/// Fail-closed: if running in-cluster and Kubernetes client initialization fails, -/// throws rather than silently falling back to a local executor. -/// -public sealed class SandboxExecutorRouter : ISandboxExecutorRouter -{ - private readonly IConfiguration _config; - private readonly ILoggerFactory _loggerFactory; - private readonly IPodNameRegistry? _podRegistry; - private readonly IAgentHostTurnTokenRegistry? _turnTokenRegistry; - private readonly Security.IRunAuthorshipCapabilityStore? _authorshipCapabilityStore; - private readonly IHttpClientFactory? _httpClientFactory; - private readonly IRunSubmittingUserResolver? _submittingUserResolver; - private readonly IGitHubTokenStore? _tokenStore; - private readonly IGitHubTokenScopeProvider? _tokenScopeProvider; - private readonly Agentweaver.Api.Auth.ISecretStore? _secretStore; - private readonly IRunEventStream? _runEventStream; - private readonly IRunOptionsStore? _runOptions; - private readonly IGitHubAccessTokenProvider? _accessTokenProvider; - private readonly Preview.ISandboxPreviewService? _previewService; - - public SandboxExecutorRouter(IConfiguration config, ILoggerFactory loggerFactory, - IPodNameRegistry? podRegistry = null, IHttpClientFactory? httpClientFactory = null, - IRunSubmittingUserResolver? submittingUserResolver = null, - IAgentHostTurnTokenRegistry? turnTokenRegistry = null, - IGitHubTokenStore? tokenStore = null, - IGitHubTokenScopeProvider? tokenScopeProvider = null, - Agentweaver.Api.Auth.ISecretStore? secretStore = null, - IRunEventStream? runEventStream = null, - IRunOptionsStore? runOptions = null, - IGitHubAccessTokenProvider? accessTokenProvider = null, - Preview.ISandboxPreviewService? previewService = null, - Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null) - { - _config = config; - _loggerFactory = loggerFactory; - _podRegistry = podRegistry; - _turnTokenRegistry = turnTokenRegistry; - _httpClientFactory = httpClientFactory; - _submittingUserResolver = submittingUserResolver; - _tokenStore = tokenStore; - _tokenScopeProvider = tokenScopeProvider; - _secretStore = secretStore; - _runEventStream = runEventStream; - _runOptions = runOptions; - _accessTokenProvider = accessTokenProvider; - _previewService = previewService; - _authorshipCapabilityStore = authorshipCapabilityStore; - } - - public ISandboxExecutor Resolve() - { - var backendOverride = _config["Sandbox:Backend"]?.ToLowerInvariant(); - var isInCluster = SandboxExecutorFactory.IsInCluster; - var logger = _loggerFactory.CreateLogger(); - - var useKubernetes = backendOverride == "kubernetes" - || (isInCluster && backendOverride != "local"); - - if (!useKubernetes) - { - logger.LogInformation( - "SandboxExecutorRouter: selecting local executor (backend={Backend}, inCluster={InCluster})", - backendOverride ?? "(none)", isInCluster); - var localExecutor = SandboxExecutorFactory.Create(logger); - if (!localExecutor.IsRealIsolation) - { - logger.LogWarning( - "⚠️ PassthroughExecutor selected — agent commands run directly on the host. Not for production use."); - } - return localExecutor; - } - - try - { - var k8sConfig = KubernetesClientConfiguration.InClusterConfig(); - var k8sClient = new Kubernetes(k8sConfig); - var sandboxOptions = new KubernetesSandboxOptions - { - Namespace = _config["Sandbox:Kubernetes:Namespace"] ?? "agentweaver", - TemplateRef = _config["Sandbox:Kubernetes:TemplateRef"] ?? "agentweaver-sandbox", - WarmPoolRef = _config["Sandbox:Kubernetes:WarmPoolRef"] - ?? _config["Sandbox:Kubernetes:TemplateRef"] ?? "agentweaver-sandbox", - AgentHostWarmPoolRef = _config["Sandbox:Kubernetes:AgentHostWarmPoolRef"] - ?? "agentweaver-agent-host", - WorkspaceMountPath = _config["Sandbox:Kubernetes:WorkspaceMountPath"] - ?? _config["Workspace:PersistentVolume:MountRoot"] - ?? _config["Workspace:Path"] - ?? "/workspace", - TimeoutSeconds = int.TryParse( - _config["Sandbox:Kubernetes:TimeoutSeconds"], out int t) ? t : 600, - ServiceCidr = _config["Sandbox:Kubernetes:ServiceCidr"] - ?? _config["Sandbox:Kubernetes:ClusterServiceCidr"], - SandboxEgressCidrExclusions = ReadSandboxEgressCidrExclusions(), - RequireMtls = !string.Equals( - _config["Sandbox:AgentHost:RequireMtls"], "false", StringComparison.OrdinalIgnoreCase), - AgentHostHealthzPath = _config["Sandbox:Kubernetes:AgentHostHealthzPath"] ?? "/healthz", - AgentHostReadyTimeoutSeconds = int.TryParse( - _config["Sandbox:Kubernetes:AgentHostReadyTimeoutSeconds"], out int rt) ? rt : 90, - AgentHostReadyPollIntervalMs = int.TryParse( - _config["Sandbox:Kubernetes:AgentHostReadyPollIntervalMs"], out int ri) ? ri : 1000, - // Option C warm-pool token fetch: same KV the API persists user tokens to. - KvUri = _config["Sandbox:AgentHost:KeyVaultUri"] - ?? _config["Auth:TokenStore:KeyVaultUri"], - }; - var k8sLogger = _loggerFactory.CreateLogger(); - WarnIfServiceCidrNotExcluded(sandboxOptions, logger); - - // Readiness gate closes the A2A cold-start race (pod Running before Kestrel binds :8088). - // Requires the named HttpClient that can reach the pod IP; skipped (null) only if no - // IHttpClientFactory was injected (which would itself be a misconfiguration in-cluster). - IAgentHostReadinessProbe? readinessProbe = null; - if (_httpClientFactory is not null) - { - readinessProbe = new HttpAgentHostReadinessProbe( - _httpClientFactory, - TimeSpan.FromSeconds(sandboxOptions.AgentHostReadyTimeoutSeconds), - TimeSpan.FromMilliseconds(sandboxOptions.AgentHostReadyPollIntervalMs), - _loggerFactory.CreateLogger()); - } - else - { - logger.LogWarning( - "SandboxExecutorRouter: no IHttpClientFactory available — AgentHost readiness gate disabled. " + - "First A2A turns may race the cold-start Kestrel bind."); - } - - logger.LogInformation( - "SandboxExecutorRouter: selecting KubernetesSandboxExecutor (namespace={Namespace}, workspaceMountPath={WorkspaceMountPath})", - sandboxOptions.Namespace, sandboxOptions.WorkspaceMountPath); - return new KubernetesSandboxExecutor( - k8sClient, sandboxOptions, k8sLogger, _podRegistry, _turnTokenRegistry, readinessProbe, - _submittingUserResolver, _httpClientFactory, _tokenStore, _secretStore, _runEventStream, - _runOptions, _accessTokenProvider, _previewService, - tokenScopeProvider: _tokenScopeProvider, - authorshipCapabilityStore: _authorshipCapabilityStore); - } - catch (Exception ex) - { - throw new InvalidOperationException( - "SandboxExecutorRouter: in-cluster Kubernetes executor initialization failed. " + - "Fail-closed: will not fall back to a local executor.", ex); - } - } - - private IReadOnlyList ReadSandboxEgressCidrExclusions() => - _config.GetSection("Sandbox:Kubernetes:SandboxEgressCidrExclusions").Get() - ?? _config.GetSection("SandboxEgressCidrExclusions").Get() - ?? []; - - private static void WarnIfServiceCidrNotExcluded( - KubernetesSandboxOptions options, - ILogger logger) - { - if (string.IsNullOrWhiteSpace(options.ServiceCidr)) - return; - - var excluded = options.SandboxEgressCidrExclusions.Any(cidr => - string.Equals(cidr.Trim(), options.ServiceCidr.Trim(), StringComparison.OrdinalIgnoreCase)); - if (!excluded) - { - logger.LogWarning( - "Sandbox egress configuration warning: cluster service CIDR {ServiceCidr} is not listed in SandboxEgressCidrExclusions. Add it to keep sandbox egress from reaching in-cluster services.", - options.ServiceCidr); - } - } -} +using k8s; +using Agentweaver.SandboxExec; +using Agentweaver.AgentRuntime.Workflow; +using Agentweaver.Api.Infrastructure; +using Agentweaver.Domain; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +namespace Agentweaver.Api.Sandbox; + +/// +/// Selects ISandboxExecutor based on: +/// 1. Sandbox:Backend config override ("kubernetes" or "local"). +/// 2. KUBERNETES_SERVICE_HOST environment variable (implicit in-cluster probe). +/// +/// Fail-closed: if running in-cluster and Kubernetes client initialization fails, +/// throws rather than silently falling back to a local executor. +/// +public sealed class SandboxExecutorRouter : ISandboxExecutorRouter +{ + private readonly IConfiguration _config; + private readonly ILoggerFactory _loggerFactory; + private readonly IPodNameRegistry? _podRegistry; + private readonly IAgentHostTurnTokenRegistry? _turnTokenRegistry; + private readonly Security.IRunAuthorshipCapabilityStore? _authorshipCapabilityStore; + private readonly IHttpClientFactory? _httpClientFactory; + private readonly IRunSubmittingUserResolver? _submittingUserResolver; + private readonly IGitHubTokenStore? _tokenStore; + private readonly IGitHubTokenScopeProvider? _tokenScopeProvider; + private readonly Agentweaver.Api.Auth.ISecretStore? _secretStore; + private readonly IRunEventStream? _runEventStream; + private readonly IRunOptionsStore? _runOptions; + private readonly IGitHubAccessTokenProvider? _accessTokenProvider; + private readonly Preview.ISandboxPreviewService? _previewService; + private readonly RunRepositoryCredentialRegistry? _repositoryCredentials; + + public SandboxExecutorRouter(IConfiguration config, ILoggerFactory loggerFactory, + IPodNameRegistry? podRegistry = null, IHttpClientFactory? httpClientFactory = null, + IRunSubmittingUserResolver? submittingUserResolver = null, + IAgentHostTurnTokenRegistry? turnTokenRegistry = null, + IGitHubTokenStore? tokenStore = null, + IGitHubTokenScopeProvider? tokenScopeProvider = null, + Agentweaver.Api.Auth.ISecretStore? secretStore = null, + IRunEventStream? runEventStream = null, + IRunOptionsStore? runOptions = null, + IGitHubAccessTokenProvider? accessTokenProvider = null, + Preview.ISandboxPreviewService? previewService = null, + RunRepositoryCredentialRegistry? repositoryCredentials = null, + Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null) + { + _config = config; + _loggerFactory = loggerFactory; + _podRegistry = podRegistry; + _turnTokenRegistry = turnTokenRegistry; + _httpClientFactory = httpClientFactory; + _submittingUserResolver = submittingUserResolver; + _tokenStore = tokenStore; + _tokenScopeProvider = tokenScopeProvider; + _secretStore = secretStore; + _runEventStream = runEventStream; + _runOptions = runOptions; + _accessTokenProvider = accessTokenProvider; + _previewService = previewService; + _repositoryCredentials = repositoryCredentials; + _authorshipCapabilityStore = authorshipCapabilityStore; + } + + public ISandboxExecutor Resolve() + { + var backendOverride = _config["Sandbox:Backend"]?.ToLowerInvariant(); + var isInCluster = SandboxExecutorFactory.IsInCluster; + var logger = _loggerFactory.CreateLogger(); + + var useKubernetes = backendOverride == "kubernetes" + || (isInCluster && backendOverride != "local"); + + if (!useKubernetes) + { + logger.LogInformation( + "SandboxExecutorRouter: selecting local executor (backend={Backend}, inCluster={InCluster})", + backendOverride ?? "(none)", isInCluster); + var localExecutor = SandboxExecutorFactory.Create(logger); + if (!localExecutor.IsRealIsolation) + { + logger.LogWarning( + "⚠️ PassthroughExecutor selected — agent commands run directly on the host. Not for production use."); + } + return localExecutor; + } + + try + { + var k8sConfig = KubernetesClientConfiguration.InClusterConfig(); + var k8sClient = new Kubernetes(k8sConfig); + var sandboxOptions = new KubernetesSandboxOptions + { + Namespace = _config["Sandbox:Kubernetes:Namespace"] ?? "agentweaver", + TemplateRef = _config["Sandbox:Kubernetes:TemplateRef"] ?? "agentweaver-sandbox", + WarmPoolRef = _config["Sandbox:Kubernetes:WarmPoolRef"] + ?? _config["Sandbox:Kubernetes:TemplateRef"] ?? "agentweaver-sandbox", + AgentHostWarmPoolRef = _config["Sandbox:Kubernetes:AgentHostWarmPoolRef"] + ?? "agentweaver-agent-host", + WorkspaceMountPath = _config["Sandbox:Kubernetes:WorkspaceMountPath"] + ?? _config["Workspace:PersistentVolume:MountRoot"] + ?? _config["Workspace:Path"] + ?? "/workspace", + TimeoutSeconds = int.TryParse( + _config["Sandbox:Kubernetes:TimeoutSeconds"], out int t) ? t : 600, + ServiceCidr = _config["Sandbox:Kubernetes:ServiceCidr"] + ?? _config["Sandbox:Kubernetes:ClusterServiceCidr"], + SandboxEgressCidrExclusions = ReadSandboxEgressCidrExclusions(), + RequireMtls = !string.Equals( + _config["Sandbox:AgentHost:RequireMtls"], "false", StringComparison.OrdinalIgnoreCase), + AgentHostHealthzPath = _config["Sandbox:Kubernetes:AgentHostHealthzPath"] ?? "/healthz", + AgentHostReadyTimeoutSeconds = int.TryParse( + _config["Sandbox:Kubernetes:AgentHostReadyTimeoutSeconds"], out int rt) ? rt : 90, + AgentHostReadyPollIntervalMs = int.TryParse( + _config["Sandbox:Kubernetes:AgentHostReadyPollIntervalMs"], out int ri) ? ri : 1000, + // Option C warm-pool token fetch: same KV the API persists user tokens to. + KvUri = _config["Sandbox:AgentHost:KeyVaultUri"] + ?? _config["Auth:TokenStore:KeyVaultUri"], + }; + var k8sLogger = _loggerFactory.CreateLogger(); + WarnIfServiceCidrNotExcluded(sandboxOptions, logger); + + // Readiness gate closes the A2A cold-start race (pod Running before Kestrel binds :8088). + // Requires the named HttpClient that can reach the pod IP; skipped (null) only if no + // IHttpClientFactory was injected (which would itself be a misconfiguration in-cluster). + IAgentHostReadinessProbe? readinessProbe = null; + if (_httpClientFactory is not null) + { + readinessProbe = new HttpAgentHostReadinessProbe( + _httpClientFactory, + TimeSpan.FromSeconds(sandboxOptions.AgentHostReadyTimeoutSeconds), + TimeSpan.FromMilliseconds(sandboxOptions.AgentHostReadyPollIntervalMs), + _loggerFactory.CreateLogger()); + } + else + { + logger.LogWarning( + "SandboxExecutorRouter: no IHttpClientFactory available — AgentHost readiness gate disabled. " + + "First A2A turns may race the cold-start Kestrel bind."); + } + + logger.LogInformation( + "SandboxExecutorRouter: selecting KubernetesSandboxExecutor (namespace={Namespace}, workspaceMountPath={WorkspaceMountPath})", + sandboxOptions.Namespace, sandboxOptions.WorkspaceMountPath); + return new KubernetesSandboxExecutor( + k8sClient, sandboxOptions, k8sLogger, _podRegistry, _turnTokenRegistry, readinessProbe, + _submittingUserResolver, _httpClientFactory, _tokenStore, _secretStore, _runEventStream, + _runOptions, _repositoryCredentials, _accessTokenProvider, _previewService, + tokenScopeProvider: _tokenScopeProvider, + authorshipCapabilityStore: _authorshipCapabilityStore); + } + catch (Exception ex) + { + throw new InvalidOperationException( + "SandboxExecutorRouter: in-cluster Kubernetes executor initialization failed. " + + "Fail-closed: will not fall back to a local executor.", ex); + } + } + + private IReadOnlyList ReadSandboxEgressCidrExclusions() => + _config.GetSection("Sandbox:Kubernetes:SandboxEgressCidrExclusions").Get() + ?? _config.GetSection("SandboxEgressCidrExclusions").Get() + ?? []; + + private static void WarnIfServiceCidrNotExcluded( + KubernetesSandboxOptions options, + ILogger logger) + { + if (string.IsNullOrWhiteSpace(options.ServiceCidr)) + return; + + var excluded = options.SandboxEgressCidrExclusions.Any(cidr => + string.Equals(cidr.Trim(), options.ServiceCidr.Trim(), StringComparison.OrdinalIgnoreCase)); + if (!excluded) + { + logger.LogWarning( + "Sandbox egress configuration warning: cluster service CIDR {ServiceCidr} is not listed in SandboxEgressCidrExclusions. Add it to keep sandbox egress from reaching in-cluster services.", + options.ServiceCidr); + } + } +} diff --git a/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs b/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs index c2a64c6ec..afc82cb04 100644 --- a/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs +++ b/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs @@ -1,549 +1,571 @@ -using System.Net.Http.Headers; -using System.Net.Http.Json; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using Agentweaver.Api.Auth; -using Agentweaver.Api.Memory; -using Microsoft.EntityFrameworkCore; -using Microsoft.IdentityModel.JsonWebTokens; -using Microsoft.IdentityModel.Tokens; - -namespace Agentweaver.Api.Webhooks; - -public enum RepoAppInstallationOutcome { Success, InstallationUnavailable, ConfigurationUnavailable, ProviderUnavailable } -internal enum RepoAppInstallationBindingOutcome { Bound, PermissionChanged, Conflict } - -internal sealed record RepoAppInstallationAuthority( - long InstallationId, - long RepositoryId, - string FullNameDisplay, - IReadOnlyDictionary Permissions); -internal sealed record RepoAppInstallationToken(string Value, DateTimeOffset? ExpiresAt); - -/// -/// API-only boundary for a short-lived Repo App JWT and the single-repository installation -/// token it mints. Neither credential is written to persistence, logs, or HTTP responses. -/// -public sealed class RepoAppInstallationTokenService( - IConfiguration configuration, - MemoryDbContext db, - ISecretStore secretStore, - IHttpClientFactory httpClientFactory) -{ - private static readonly TimeSpan JwtLifetime = TimeSpan.FromMinutes(9); - private static readonly IReadOnlyDictionary UnattendedRepositoryPermissionCeilings = - new Dictionary(StringComparer.Ordinal) - { - ["contents"] = "write", - ["pull_requests"] = "write", - }; - private static readonly IReadOnlyDictionary RepositoryMetadataPermissionScope = - new Dictionary(StringComparer.Ordinal) - { - ["metadata"] = "read", - }; - - public async Task MintForRepositoryAsync( - long installationId, - long repositoryId, - Func useToken, - CancellationToken ct = default) - { - if (installationId <= 0 || repositoryId <= 0) - return RepoAppInstallationOutcome.InstallationUnavailable; - - var installationActive = await db.GitHubInstallations.AsNoTracking() - .AnyAsync(x => x.InstallationId == installationId && - x.AppKind == GitHubAppKind.Repo && - x.RevokedAt == null, ct).ConfigureAwait(false); - var grant = await db.GitHubRepositoryGrants.AsNoTracking() - .SingleOrDefaultAsync(x => x.InstallationId == installationId && - x.RepositoryId == repositoryId && - x.RevokedAt == null, ct).ConfigureAwait(false); - if (!installationActive || grant is null) - return RepoAppInstallationOutcome.InstallationUnavailable; - - var authority = await GetRepositoryAuthorityAsync(installationId, repositoryId, ct).ConfigureAwait(false); - if (authority is null) - return RepoAppInstallationOutcome.ProviderUnavailable; - if (!CryptographicOperations.FixedTimeEquals( - Encoding.UTF8.GetBytes(grant.PermissionDigest), - Encoding.UTF8.GetBytes(CreatePermissionDigest(authority.Permissions)))) - { - await new RepoAppInstallationLifecycleService(db) - .InvalidateForPermissionChangeAsync(installationId, repositoryId, ct).ConfigureAwait(false); - return RepoAppInstallationOutcome.InstallationUnavailable; - } - if (!TryCreateUnattendedPermissionScope(authority.Permissions, out var requestedPermissions)) - return RepoAppInstallationOutcome.InstallationUnavailable; - - try - { - var appJwt = await CreateAppJwtAsync(ct).ConfigureAwait(false); - if (appJwt is null) - return RepoAppInstallationOutcome.ConfigurationUnavailable; - var installationToken = await GetInstallationTokenAsync( - appJwt, installationId, repositoryId, requestedPermissions, ct).ConfigureAwait(false); - if (installationToken is null) - return RepoAppInstallationOutcome.ProviderUnavailable; - - if (installationToken.ExpiresAt is null || installationToken.ExpiresAt <= DateTimeOffset.UtcNow) - return RepoAppInstallationOutcome.ProviderUnavailable; - await useToken(installationToken.Value, installationToken.ExpiresAt.Value).ConfigureAwait(false); - return RepoAppInstallationOutcome.Success; - } - catch (OperationCanceledException) when (!ct.IsCancellationRequested) - { - return RepoAppInstallationOutcome.ProviderUnavailable; - } - catch (HttpRequestException) - { - return RepoAppInstallationOutcome.ProviderUnavailable; - } - } - - public async Task VerifyRepositoryInstallationAsync( - long installationId, - long repositoryId, - CancellationToken ct = default) - => await GetRepositoryAuthorityAsync(installationId, repositoryId, ct).ConfigureAwait(false) is not null; - - /// - /// Resolves the installation's exact repository authority from GitHub. The request supplies - /// only numeric identifiers; permissions and the display name are provider-owned values. - /// - internal async Task GetRepositoryAuthorityAsync( - long installationId, - long repositoryId, - CancellationToken ct = default) - { - if (installationId <= 0 || repositoryId <= 0) - return null; - - var appJwt = await CreateAppJwtAsync(ct).ConfigureAwait(false); - if (appJwt is null) - return null; - - try - { - using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); - timeout.CancelAfter(TimeSpan.FromSeconds(10)); - var client = httpClientFactory.CreateClient("github"); - using var installationRequest = CreateGitHubRequest( - HttpMethod.Get, $"/repositories/{repositoryId}/installation", appJwt); - using var installationResponse = await client.SendAsync(installationRequest, timeout.Token).ConfigureAwait(false); - if (!installationResponse.IsSuccessStatusCode) - return null; - using var installationDocument = JsonDocument.Parse( - await installationResponse.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); - var installation = installationDocument.RootElement; - if (!installation.TryGetProperty("id", out var actualInstallation) || - !actualInstallation.TryGetInt64(out var actualInstallationId) || - actualInstallationId != installationId || - !installation.TryGetProperty("repository_selection", out var repositorySelection) || - repositorySelection.ValueKind != JsonValueKind.String || - string.IsNullOrWhiteSpace(repositorySelection.GetString()) || - !installation.TryGetProperty("account", out var account) || - account.ValueKind != JsonValueKind.Object || - !TryGetNormalizedPermissions(installation, out var permissions)) - return null; - - var metadataToken = await GetInstallationTokenAsync( - appJwt, installationId, repositoryId, RepositoryMetadataPermissionScope, timeout.Token) - .ConfigureAwait(false); - if (metadataToken is null) - return null; - using var repositoryRequest = CreateGitHubRequest( - HttpMethod.Get, $"/repositories/{repositoryId}", metadataToken.Value); - using var repositoryResponse = await client.SendAsync(repositoryRequest, timeout.Token).ConfigureAwait(false); - if (!repositoryResponse.IsSuccessStatusCode) - return null; - using var repositoryDocument = JsonDocument.Parse( - await repositoryResponse.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); - var repository = repositoryDocument.RootElement; - if (!repository.TryGetProperty("id", out var actualRepository) || - !actualRepository.TryGetInt64(out var actualRepositoryId) || - actualRepositoryId != repositoryId || - !repository.TryGetProperty("full_name", out var fullName) || - fullName.ValueKind != JsonValueKind.String || - string.IsNullOrWhiteSpace(fullName.GetString())) - return null; - - return new RepoAppInstallationAuthority( - installationId, repositoryId, fullName.GetString()!, permissions); - } - catch (OperationCanceledException) when (!ct.IsCancellationRequested) - { - return null; - } - catch (HttpRequestException) - { - return null; - } - catch (JsonException) - { - return null; - } - } - - private async Task CreateAppJwtAsync(CancellationToken ct) - { - if (!long.TryParse(configuration["Auth:RepoApp:AppId"], out var appId) || appId <= 0 || - string.IsNullOrWhiteSpace(configuration["Auth:RepoApp:PrivateKeySecretName"])) - return null; - var pem = await secretStore.GetSecretAsync(configuration["Auth:RepoApp:PrivateKeySecretName"]!, ct) - .ConfigureAwait(false); - if (!pem.Found || string.IsNullOrWhiteSpace(pem.Value)) - return null; - try - { - return CreateAppJwt(appId, pem.Value); - } - catch (CryptographicException) - { - return null; - } - } - - private HttpRequestMessage CreateGitHubRequest(HttpMethod method, string path, string appJwt) - { - var request = new HttpRequestMessage( - method, $"{(configuration["Auth:RepoApp:ApiUrl"] ?? "https://api.github.com").TrimEnd('/')}{path}"); - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", appJwt); - request.Headers.UserAgent.ParseAdd("Agentweaver/1.0"); - request.Headers.Accept.ParseAdd("application/vnd.github+json"); - return request; - } - - private async Task GetInstallationTokenAsync( - string appJwt, - long installationId, - long repositoryId, - IReadOnlyDictionary permissions, - CancellationToken ct) - { - using var request = CreateGitHubRequest( - HttpMethod.Post, $"/app/installations/{installationId}/access_tokens", appJwt); - request.Content = JsonContent.Create(new { repository_ids = new[] { repositoryId }, permissions }); - using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); - timeout.CancelAfter(TimeSpan.FromSeconds(10)); - using var response = await httpClientFactory.CreateClient("github").SendAsync(request, timeout.Token) - .ConfigureAwait(false); - if (!response.IsSuccessStatusCode) - return null; - using var document = JsonDocument.Parse( - await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); - if (!document.RootElement.TryGetProperty("token", out var token) || - string.IsNullOrWhiteSpace(token.GetString())) - return null; - DateTimeOffset? expiresAt = document.RootElement.TryGetProperty("expires_at", out var expiresAtElement) && - DateTimeOffset.TryParse(expiresAtElement.GetString(), out var parsedExpiry) - ? parsedExpiry - : null; - return new(token.GetString()!, expiresAt); - } - - private static bool TryGetNormalizedPermissions( - JsonElement installation, - out IReadOnlyDictionary permissions) - { - permissions = new Dictionary(); - if (!installation.TryGetProperty("permissions", out var source) || - source.ValueKind != JsonValueKind.Object) - return false; - - var normalized = new Dictionary(StringComparer.Ordinal); - foreach (var permission in source.EnumerateObject()) - { - if (permission.Value.ValueKind != JsonValueKind.String) - return false; - var name = permission.Name.Trim().ToLowerInvariant(); - var value = permission.Value.GetString()?.Trim().ToLowerInvariant(); - if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(value) || - !normalized.TryAdd(name, value)) - return false; - } - permissions = normalized; - return normalized.Count > 0; - } - - private static bool TryCreateUnattendedPermissionScope( - IReadOnlyDictionary providerPermissions, - out IReadOnlyDictionary requestedPermissions) - { - var requested = new Dictionary(StringComparer.Ordinal); - foreach (var ceiling in UnattendedRepositoryPermissionCeilings) - { - if (!providerPermissions.TryGetValue(ceiling.Key, out var actual)) - continue; - if (!string.Equals(actual, "read", StringComparison.Ordinal) && - !string.Equals(actual, "write", StringComparison.Ordinal)) - { - requestedPermissions = new Dictionary(); - return false; - } - if (string.Equals(ceiling.Value, "read", StringComparison.Ordinal) && - string.Equals(actual, "write", StringComparison.Ordinal)) - { - requestedPermissions = new Dictionary(); - return false; - } - requested[ceiling.Key] = actual; - } - requestedPermissions = requested; - return requested.Count > 0; - } - - internal static string CreateAppJwt(long appId, string pem) - { - using var rsa = RSA.Create(); - rsa.ImportFromPem(pem); - var now = DateTime.UtcNow; - var signingKey = new RsaSecurityKey(rsa) - { - CryptoProviderFactory = new CryptoProviderFactory { CacheSignatureProviders = false }, - }; - return new JsonWebTokenHandler().CreateToken(new SecurityTokenDescriptor - { - Issuer = appId.ToString(System.Globalization.CultureInfo.InvariantCulture), - IssuedAt = now.AddMinutes(-1), - NotBefore = now.AddMinutes(-1), - Expires = now.Add(JwtLifetime), - SigningCredentials = new SigningCredentials(signingKey, SecurityAlgorithms.RsaSha256), - }); - } - - internal static string CreatePermissionDigest(IReadOnlyDictionary permissions) - { - var canonical = string.Join("&", permissions.OrderBy(x => x.Key, StringComparer.Ordinal) - .Select(x => $"{x.Key.Trim().ToLowerInvariant()}={x.Value.Trim().ToLowerInvariant()}")); - return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(canonical))).ToLowerInvariant(); - } -} - -/// Durable installation/grant state machine for authenticated Repo App deliveries. -public sealed class RepoAppInstallationLifecycleService(MemoryDbContext db) -{ - private const string CompletedEventPrefix = "completed/"; - private static readonly TimeSpan ProcessingLease = TimeSpan.FromMinutes(10); - - public async Task<(bool Claimed, IReadOnlyList ProjectIds)> ProcessAsync( - string deliveryId, - string eventName, - GitHubWebhookPayload payload, - CancellationToken ct = default) - { - if (string.IsNullOrWhiteSpace(deliveryId)) - return (false, []); - - db.ChangeTracker.Clear(); - await using var transaction = await db.Database.BeginTransactionAsync(ct).ConfigureAwait(false); - db.GitHubLifecycleDeliveries.Add(new GitHubLifecycleDeliveryRecord - { - DeliveryId = deliveryId, - EventName = eventName, - InstallationId = payload.Installation?.Id, - RepositoryId = payload.Repository?.Id, - ReceivedAt = DateTimeOffset.UtcNow, - }); - try - { - await db.SaveChangesAsync(ct).ConfigureAwait(false); - } - catch (DbUpdateException) - { - await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); - db.ChangeTracker.Clear(); - var leaseExpiresBefore = DateTimeOffset.UtcNow.Subtract(ProcessingLease); - var abandoned = await db.GitHubLifecycleDeliveries.FindAsync([deliveryId], ct).ConfigureAwait(false); - if (abandoned is null || abandoned.EventName != eventName || abandoned.ReceivedAt >= leaseExpiresBefore) - return (false, []); - var reclaimed = await db.GitHubLifecycleDeliveries - .Where(x => x.DeliveryId == deliveryId && - x.EventName == abandoned.EventName && - x.ReceivedAt == abandoned.ReceivedAt) - .ExecuteDeleteAsync(ct).ConfigureAwait(false); - if (reclaimed != 1) - return (false, []); - return await ProcessAsync(deliveryId, eventName, payload, ct).ConfigureAwait(false); - } - - var installationId = (payload.Installation?.Id).GetValueOrDefault(); - if (installationId > 0 && eventName is "installation" or "installation_repositories") - { - await ApplyLifecycleAsync(installationId, payload, ct).ConfigureAwait(false); - await db.SaveChangesAsync(ct).ConfigureAwait(false); - } - - db.ChangeTracker.Clear(); - var installationActive = installationId > 0 && await db.GitHubInstallations.AsNoTracking() - .AnyAsync(x => x.InstallationId == installationId && - x.AppKind == GitHubAppKind.Repo && - x.RevokedAt == null, ct).ConfigureAwait(false); - var projectIds = installationActive && payload.Repository?.Id is > 0 - ? await db.GitHubRepositoryGrants.AsNoTracking() - .Where(x => x.InstallationId == installationId && - x.RepositoryId == payload.Repository.Id && - x.RevokedAt == null) - .Select(x => x.ProjectId).ToListAsync(ct).ConfigureAwait(false) - : []; - await db.SaveChangesAsync(ct).ConfigureAwait(false); - await transaction.CommitAsync(ct).ConfigureAwait(false); - return (true, projectIds); - } - - /// - /// Releases a claim only when downstream dispatch did not complete, allowing GitHub to retry. - /// The dispatch path has its own delivery-id idempotency guard. - /// - public async Task ReleaseAsync(string deliveryId, CancellationToken ct = default) - { - db.ChangeTracker.Clear(); - await db.GitHubLifecycleDeliveries.Where(x => x.DeliveryId == deliveryId) - .ExecuteDeleteAsync(ct).ConfigureAwait(false); - } - - public Task IsCompletedAsync(string deliveryId, CancellationToken ct = default) => - db.GitHubLifecycleDeliveries.AsNoTracking() - .AnyAsync(x => x.DeliveryId == deliveryId && - x.EventName.StartsWith(CompletedEventPrefix), ct); - - public async Task CompleteAsync(string deliveryId, CancellationToken ct = default) - { - db.ChangeTracker.Clear(); - var current = await db.GitHubLifecycleDeliveries.AsNoTracking() - .Where(x => x.DeliveryId == deliveryId) - .Select(x => x.EventName).SingleOrDefaultAsync(ct).ConfigureAwait(false); - if (current is null) - return false; - if (current.StartsWith(CompletedEventPrefix, StringComparison.Ordinal)) - return true; - return await db.GitHubLifecycleDeliveries.Where(x => x.DeliveryId == deliveryId && x.EventName == current) - .ExecuteUpdateAsync(s => s.SetProperty(x => x.EventName, $"{CompletedEventPrefix}{current}"), ct) - .ConfigureAwait(false) == 1; - } - - internal async Task BindAsync( - string projectId, - RepoAppInstallationAuthority authority, - CancellationToken ct = default) - { - await using var transaction = await db.Database.BeginTransactionAsync( - System.Data.IsolationLevel.Serializable, ct).ConfigureAwait(false); - var now = DateTimeOffset.UtcNow; - var installation = await db.GitHubInstallations.FindAsync([authority.InstallationId], ct).ConfigureAwait(false); - if (installation is not null && installation.ProjectId is not null && - !string.Equals(installation.ProjectId, projectId, StringComparison.Ordinal)) - return RepoAppInstallationBindingOutcome.Conflict; - if (installation is null) - db.GitHubInstallations.Add(new GitHubInstallationRecord - { - InstallationId = authority.InstallationId, AppKind = GitHubAppKind.Repo, ProjectId = projectId, CreatedAt = now, - }); - else - { - installation.ProjectId = projectId; - installation.RevokedAt = null; - } - - var grant = await db.GitHubRepositoryGrants.FindAsync( - [authority.InstallationId, authority.RepositoryId], ct).ConfigureAwait(false); - if (grant is not null && !string.Equals(grant.ProjectId, projectId, StringComparison.Ordinal)) - return RepoAppInstallationBindingOutcome.Conflict; - if (grant is null) - db.GitHubRepositoryGrants.Add(new GitHubRepositoryGrantRecord - { - InstallationId = authority.InstallationId, RepositoryId = authority.RepositoryId, ProjectId = projectId, - FullNameDisplay = authority.FullNameDisplay, - PermissionDigest = RepoAppInstallationTokenService.CreatePermissionDigest(authority.Permissions), - GrantedAt = now, - }); - else - { - var permissionDigest = RepoAppInstallationTokenService.CreatePermissionDigest(authority.Permissions); - if (!CryptographicOperations.FixedTimeEquals( - Encoding.UTF8.GetBytes(grant.PermissionDigest), Encoding.UTF8.GetBytes(permissionDigest))) - { - grant.FullNameDisplay = authority.FullNameDisplay; - grant.RevokedAt = now; - await InvalidateForPermissionChangeAsync(authority.InstallationId, authority.RepositoryId, ct) - .ConfigureAwait(false); - await db.SaveChangesAsync(ct).ConfigureAwait(false); - await transaction.CommitAsync(ct).ConfigureAwait(false); - return RepoAppInstallationBindingOutcome.PermissionChanged; - } - grant.FullNameDisplay = authority.FullNameDisplay; - grant.RevokedAt = null; - } - try - { - await db.SaveChangesAsync(ct).ConfigureAwait(false); - await transaction.CommitAsync(ct).ConfigureAwait(false); - return RepoAppInstallationBindingOutcome.Bound; - } - catch (DbUpdateException) - { - db.ChangeTracker.Clear(); - return RepoAppInstallationBindingOutcome.Conflict; - } - } - - public async Task InvalidateForPermissionChangeAsync( - long installationId, - long repositoryId, - CancellationToken ct = default) - { - var transaction = db.Database.CurrentTransaction is null - ? await db.Database.BeginTransactionAsync(ct).ConfigureAwait(false) - : null; - var now = DateTimeOffset.UtcNow; - try - { - await db.GitHubRepositoryGrants - .Where(x => x.InstallationId == installationId && - x.RepositoryId == repositoryId && - x.RevokedAt == null) - .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); - await db.AutomationActivations - .Where(x => x.InstallationId == installationId && - x.RepositoryId == repositoryId && - x.Status != AutomationActivationStatus.Invalidated) - .ExecuteUpdateAsync(s => s - .SetProperty(x => x.Status, AutomationActivationStatus.Invalidated) - .SetProperty(x => x.InvalidatedAt, now), ct).ConfigureAwait(false); - if (transaction is not null) - await transaction.CommitAsync(ct).ConfigureAwait(false); - } - finally - { - if (transaction is not null) - await transaction.DisposeAsync().ConfigureAwait(false); - } - } - - private async Task ApplyLifecycleAsync(long installationId, GitHubWebhookPayload payload, CancellationToken ct) - { - var installation = await db.GitHubInstallations.FindAsync([installationId], ct).ConfigureAwait(false); - if (installation is null) - return; // A delivery can never create a project binding from untrusted display data. - - var now = DateTimeOffset.UtcNow; - if (payload.Action is "deleted" or "suspend") - { - installation.RevokedAt = now; - await db.GitHubRepositoryGrants.Where(x => x.InstallationId == installationId && x.RevokedAt == null) - .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); - return; - } - if (payload.Action is "created" or "unsuspend") - installation.RevokedAt = null; - - foreach (var repository in payload.RepositoriesRemoved ?? []) - { - if (repository.Id > 0) - await db.GitHubRepositoryGrants.Where(x => x.InstallationId == installationId && x.RepositoryId == repository.Id) - .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); - } - } -} +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Agentweaver.Api.Auth; +using Agentweaver.Api.Memory; +using Microsoft.EntityFrameworkCore; +using Microsoft.IdentityModel.JsonWebTokens; +using Microsoft.IdentityModel.Tokens; + +namespace Agentweaver.Api.Webhooks; + +public enum RepoAppInstallationOutcome { Success, InstallationUnavailable, ConfigurationUnavailable, ProviderUnavailable } +internal enum RepoAppInstallationBindingOutcome { Bound, PermissionChanged, Conflict } + +internal sealed record RepoAppInstallationAuthority( + long InstallationId, + long RepositoryId, + string FullNameDisplay, + IReadOnlyDictionary Permissions); +internal sealed record RepoAppInstallationToken(string Value, DateTimeOffset? ExpiresAt); + +/// +/// API-only boundary for a short-lived Repo App JWT and the single-repository installation +/// token it mints. Neither credential is written to persistence, logs, or HTTP responses. +/// +public sealed class RepoAppInstallationTokenService( + IConfiguration configuration, + MemoryDbContext db, + ISecretStore secretStore, + IHttpClientFactory httpClientFactory) +{ + private static readonly TimeSpan JwtLifetime = TimeSpan.FromMinutes(9); + private static readonly IReadOnlyDictionary UnattendedRepositoryPermissionCeilings = + new Dictionary(StringComparer.Ordinal) + { + ["contents"] = "write", + ["pull_requests"] = "write", + }; + private static readonly IReadOnlyDictionary RepositoryMetadataPermissionScope = + new Dictionary(StringComparer.Ordinal) + { + ["metadata"] = "read", + }; + + public async Task MintForRepositoryAsync( + long installationId, + long repositoryId, + Func useToken, + CancellationToken ct = default) + { + if (installationId <= 0 || repositoryId <= 0) + return RepoAppInstallationOutcome.InstallationUnavailable; + + var installationActive = await db.GitHubInstallations.AsNoTracking() + .AnyAsync(x => x.InstallationId == installationId && + x.AppKind == GitHubAppKind.Repo && + x.RevokedAt == null, ct).ConfigureAwait(false); + var grant = await db.GitHubRepositoryGrants.AsNoTracking() + .SingleOrDefaultAsync(x => x.InstallationId == installationId && + x.RepositoryId == repositoryId && + x.RevokedAt == null, ct).ConfigureAwait(false); + if (!installationActive || grant is null) + return RepoAppInstallationOutcome.InstallationUnavailable; + + var authority = await GetRepositoryAuthorityAsync(installationId, repositoryId, ct).ConfigureAwait(false); + if (authority is null) + return RepoAppInstallationOutcome.ProviderUnavailable; + if (!CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(grant.PermissionDigest), + Encoding.UTF8.GetBytes(CreatePermissionDigest(authority.Permissions)))) + { + await new RepoAppInstallationLifecycleService(db) + .InvalidateForPermissionChangeAsync(installationId, repositoryId, ct).ConfigureAwait(false); + return RepoAppInstallationOutcome.InstallationUnavailable; + } + if (!TryCreateUnattendedPermissionScope(authority.Permissions, out var requestedPermissions)) + return RepoAppInstallationOutcome.InstallationUnavailable; + + try + { + var appJwt = await CreateAppJwtAsync(ct).ConfigureAwait(false); + if (appJwt is null) + return RepoAppInstallationOutcome.ConfigurationUnavailable; + var installationToken = await GetInstallationTokenAsync( + appJwt, installationId, repositoryId, requestedPermissions, ct).ConfigureAwait(false); + if (installationToken is null) + return RepoAppInstallationOutcome.ProviderUnavailable; + + if (installationToken.ExpiresAt is null || installationToken.ExpiresAt <= DateTimeOffset.UtcNow) + return RepoAppInstallationOutcome.ProviderUnavailable; + await useToken(installationToken.Value, installationToken.ExpiresAt.Value).ConfigureAwait(false); + return RepoAppInstallationOutcome.Success; + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return RepoAppInstallationOutcome.ProviderUnavailable; + } + catch (HttpRequestException) + { + return RepoAppInstallationOutcome.ProviderUnavailable; + } + } + + public async Task VerifyRepositoryInstallationAsync( + long installationId, + long repositoryId, + CancellationToken ct = default) + => await GetRepositoryAuthorityAsync(installationId, repositoryId, ct).ConfigureAwait(false) is not null; + + /// Revokes a minted installation credential. This method does not persist or log it. + public async Task RevokeRepositoryTokenAsync(string token, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(token)) + return; + + try + { + using var request = CreateGitHubRequest(HttpMethod.Delete, "/installation/token", token); + using var response = await httpClientFactory.CreateClient("github").SendAsync(request, ct) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (HttpRequestException) + { + // Token expiry is the backstop. Release and orphan cleanup must not fail on revoke. + } + } + + /// + /// Resolves the installation's exact repository authority from GitHub. The request supplies + /// only numeric identifiers; permissions and the display name are provider-owned values. + /// + internal async Task GetRepositoryAuthorityAsync( + long installationId, + long repositoryId, + CancellationToken ct = default) + { + if (installationId <= 0 || repositoryId <= 0) + return null; + + var appJwt = await CreateAppJwtAsync(ct).ConfigureAwait(false); + if (appJwt is null) + return null; + + try + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeout.CancelAfter(TimeSpan.FromSeconds(10)); + var client = httpClientFactory.CreateClient("github"); + using var installationRequest = CreateGitHubRequest( + HttpMethod.Get, $"/repositories/{repositoryId}/installation", appJwt); + using var installationResponse = await client.SendAsync(installationRequest, timeout.Token).ConfigureAwait(false); + if (!installationResponse.IsSuccessStatusCode) + return null; + using var installationDocument = JsonDocument.Parse( + await installationResponse.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); + var installation = installationDocument.RootElement; + if (!installation.TryGetProperty("id", out var actualInstallation) || + !actualInstallation.TryGetInt64(out var actualInstallationId) || + actualInstallationId != installationId || + !installation.TryGetProperty("repository_selection", out var repositorySelection) || + repositorySelection.ValueKind != JsonValueKind.String || + string.IsNullOrWhiteSpace(repositorySelection.GetString()) || + !installation.TryGetProperty("account", out var account) || + account.ValueKind != JsonValueKind.Object || + !TryGetNormalizedPermissions(installation, out var permissions)) + return null; + + var metadataToken = await GetInstallationTokenAsync( + appJwt, installationId, repositoryId, RepositoryMetadataPermissionScope, timeout.Token) + .ConfigureAwait(false); + if (metadataToken is null) + return null; + using var repositoryRequest = CreateGitHubRequest( + HttpMethod.Get, $"/repositories/{repositoryId}", metadataToken.Value); + using var repositoryResponse = await client.SendAsync(repositoryRequest, timeout.Token).ConfigureAwait(false); + if (!repositoryResponse.IsSuccessStatusCode) + return null; + using var repositoryDocument = JsonDocument.Parse( + await repositoryResponse.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); + var repository = repositoryDocument.RootElement; + if (!repository.TryGetProperty("id", out var actualRepository) || + !actualRepository.TryGetInt64(out var actualRepositoryId) || + actualRepositoryId != repositoryId || + !repository.TryGetProperty("full_name", out var fullName) || + fullName.ValueKind != JsonValueKind.String || + string.IsNullOrWhiteSpace(fullName.GetString())) + return null; + + return new RepoAppInstallationAuthority( + installationId, repositoryId, fullName.GetString()!, permissions); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return null; + } + catch (HttpRequestException) + { + return null; + } + catch (JsonException) + { + return null; + } + } + + private async Task CreateAppJwtAsync(CancellationToken ct) + { + if (!long.TryParse(configuration["Auth:RepoApp:AppId"], out var appId) || appId <= 0 || + string.IsNullOrWhiteSpace(configuration["Auth:RepoApp:PrivateKeySecretName"])) + return null; + var pem = await secretStore.GetSecretAsync(configuration["Auth:RepoApp:PrivateKeySecretName"]!, ct) + .ConfigureAwait(false); + if (!pem.Found || string.IsNullOrWhiteSpace(pem.Value)) + return null; + try + { + return CreateAppJwt(appId, pem.Value); + } + catch (CryptographicException) + { + return null; + } + } + + private HttpRequestMessage CreateGitHubRequest(HttpMethod method, string path, string appJwt) + { + var request = new HttpRequestMessage( + method, $"{(configuration["Auth:RepoApp:ApiUrl"] ?? "https://api.github.com").TrimEnd('/')}{path}"); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", appJwt); + request.Headers.UserAgent.ParseAdd("Agentweaver/1.0"); + request.Headers.Accept.ParseAdd("application/vnd.github+json"); + return request; + } + + private async Task GetInstallationTokenAsync( + string appJwt, + long installationId, + long repositoryId, + IReadOnlyDictionary permissions, + CancellationToken ct) + { + using var request = CreateGitHubRequest( + HttpMethod.Post, $"/app/installations/{installationId}/access_tokens", appJwt); + request.Content = JsonContent.Create(new { repository_ids = new[] { repositoryId }, permissions }); + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeout.CancelAfter(TimeSpan.FromSeconds(10)); + using var response = await httpClientFactory.CreateClient("github").SendAsync(request, timeout.Token) + .ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + return null; + using var document = JsonDocument.Parse( + await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); + if (!document.RootElement.TryGetProperty("token", out var token) || + string.IsNullOrWhiteSpace(token.GetString())) + return null; + DateTimeOffset? expiresAt = document.RootElement.TryGetProperty("expires_at", out var expiresAtElement) && + DateTimeOffset.TryParse(expiresAtElement.GetString(), out var parsedExpiry) + ? parsedExpiry + : null; + return new(token.GetString()!, expiresAt); + } + + private static bool TryGetNormalizedPermissions( + JsonElement installation, + out IReadOnlyDictionary permissions) + { + permissions = new Dictionary(); + if (!installation.TryGetProperty("permissions", out var source) || + source.ValueKind != JsonValueKind.Object) + return false; + + var normalized = new Dictionary(StringComparer.Ordinal); + foreach (var permission in source.EnumerateObject()) + { + if (permission.Value.ValueKind != JsonValueKind.String) + return false; + var name = permission.Name.Trim().ToLowerInvariant(); + var value = permission.Value.GetString()?.Trim().ToLowerInvariant(); + if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(value) || + !normalized.TryAdd(name, value)) + return false; + } + permissions = normalized; + return normalized.Count > 0; + } + + private static bool TryCreateUnattendedPermissionScope( + IReadOnlyDictionary providerPermissions, + out IReadOnlyDictionary requestedPermissions) + { + var requested = new Dictionary(StringComparer.Ordinal); + foreach (var ceiling in UnattendedRepositoryPermissionCeilings) + { + if (!providerPermissions.TryGetValue(ceiling.Key, out var actual)) + continue; + if (!string.Equals(actual, "read", StringComparison.Ordinal) && + !string.Equals(actual, "write", StringComparison.Ordinal)) + { + requestedPermissions = new Dictionary(); + return false; + } + if (string.Equals(ceiling.Value, "read", StringComparison.Ordinal) && + string.Equals(actual, "write", StringComparison.Ordinal)) + { + requestedPermissions = new Dictionary(); + return false; + } + requested[ceiling.Key] = actual; + } + requestedPermissions = requested; + return requested.Count > 0; + } + + internal static string CreateAppJwt(long appId, string pem) + { + using var rsa = RSA.Create(); + rsa.ImportFromPem(pem); + var now = DateTime.UtcNow; + var signingKey = new RsaSecurityKey(rsa) + { + CryptoProviderFactory = new CryptoProviderFactory { CacheSignatureProviders = false }, + }; + return new JsonWebTokenHandler().CreateToken(new SecurityTokenDescriptor + { + Issuer = appId.ToString(System.Globalization.CultureInfo.InvariantCulture), + IssuedAt = now.AddMinutes(-1), + NotBefore = now.AddMinutes(-1), + Expires = now.Add(JwtLifetime), + SigningCredentials = new SigningCredentials(signingKey, SecurityAlgorithms.RsaSha256), + }); + } + + internal static string CreatePermissionDigest(IReadOnlyDictionary permissions) + { + var canonical = string.Join("&", permissions.OrderBy(x => x.Key, StringComparer.Ordinal) + .Select(x => $"{x.Key.Trim().ToLowerInvariant()}={x.Value.Trim().ToLowerInvariant()}")); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(canonical))).ToLowerInvariant(); + } +} + +/// Durable installation/grant state machine for authenticated Repo App deliveries. +public sealed class RepoAppInstallationLifecycleService(MemoryDbContext db) +{ + private const string CompletedEventPrefix = "completed/"; + private static readonly TimeSpan ProcessingLease = TimeSpan.FromMinutes(10); + + public async Task<(bool Claimed, IReadOnlyList ProjectIds)> ProcessAsync( + string deliveryId, + string eventName, + GitHubWebhookPayload payload, + CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(deliveryId)) + return (false, []); + + db.ChangeTracker.Clear(); + await using var transaction = await db.Database.BeginTransactionAsync(ct).ConfigureAwait(false); + db.GitHubLifecycleDeliveries.Add(new GitHubLifecycleDeliveryRecord + { + DeliveryId = deliveryId, + EventName = eventName, + InstallationId = payload.Installation?.Id, + RepositoryId = payload.Repository?.Id, + ReceivedAt = DateTimeOffset.UtcNow, + }); + try + { + await db.SaveChangesAsync(ct).ConfigureAwait(false); + } + catch (DbUpdateException) + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + db.ChangeTracker.Clear(); + var leaseExpiresBefore = DateTimeOffset.UtcNow.Subtract(ProcessingLease); + var abandoned = await db.GitHubLifecycleDeliveries.FindAsync([deliveryId], ct).ConfigureAwait(false); + if (abandoned is null || abandoned.EventName != eventName || abandoned.ReceivedAt >= leaseExpiresBefore) + return (false, []); + var reclaimed = await db.GitHubLifecycleDeliveries + .Where(x => x.DeliveryId == deliveryId && + x.EventName == abandoned.EventName && + x.ReceivedAt == abandoned.ReceivedAt) + .ExecuteDeleteAsync(ct).ConfigureAwait(false); + if (reclaimed != 1) + return (false, []); + return await ProcessAsync(deliveryId, eventName, payload, ct).ConfigureAwait(false); + } + + var installationId = (payload.Installation?.Id).GetValueOrDefault(); + if (installationId > 0 && eventName is "installation" or "installation_repositories") + { + await ApplyLifecycleAsync(installationId, payload, ct).ConfigureAwait(false); + await db.SaveChangesAsync(ct).ConfigureAwait(false); + } + + db.ChangeTracker.Clear(); + var installationActive = installationId > 0 && await db.GitHubInstallations.AsNoTracking() + .AnyAsync(x => x.InstallationId == installationId && + x.AppKind == GitHubAppKind.Repo && + x.RevokedAt == null, ct).ConfigureAwait(false); + var projectIds = installationActive && payload.Repository?.Id is > 0 + ? await db.GitHubRepositoryGrants.AsNoTracking() + .Where(x => x.InstallationId == installationId && + x.RepositoryId == payload.Repository.Id && + x.RevokedAt == null) + .Select(x => x.ProjectId).ToListAsync(ct).ConfigureAwait(false) + : []; + await db.SaveChangesAsync(ct).ConfigureAwait(false); + await transaction.CommitAsync(ct).ConfigureAwait(false); + return (true, projectIds); + } + + /// + /// Releases a claim only when downstream dispatch did not complete, allowing GitHub to retry. + /// The dispatch path has its own delivery-id idempotency guard. + /// + public async Task ReleaseAsync(string deliveryId, CancellationToken ct = default) + { + db.ChangeTracker.Clear(); + await db.GitHubLifecycleDeliveries.Where(x => x.DeliveryId == deliveryId) + .ExecuteDeleteAsync(ct).ConfigureAwait(false); + } + + public Task IsCompletedAsync(string deliveryId, CancellationToken ct = default) => + db.GitHubLifecycleDeliveries.AsNoTracking() + .AnyAsync(x => x.DeliveryId == deliveryId && + x.EventName.StartsWith(CompletedEventPrefix), ct); + + public async Task CompleteAsync(string deliveryId, CancellationToken ct = default) + { + db.ChangeTracker.Clear(); + var current = await db.GitHubLifecycleDeliveries.AsNoTracking() + .Where(x => x.DeliveryId == deliveryId) + .Select(x => x.EventName).SingleOrDefaultAsync(ct).ConfigureAwait(false); + if (current is null) + return false; + if (current.StartsWith(CompletedEventPrefix, StringComparison.Ordinal)) + return true; + return await db.GitHubLifecycleDeliveries.Where(x => x.DeliveryId == deliveryId && x.EventName == current) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.EventName, $"{CompletedEventPrefix}{current}"), ct) + .ConfigureAwait(false) == 1; + } + + internal async Task BindAsync( + string projectId, + RepoAppInstallationAuthority authority, + CancellationToken ct = default) + { + await using var transaction = await db.Database.BeginTransactionAsync( + System.Data.IsolationLevel.Serializable, ct).ConfigureAwait(false); + var now = DateTimeOffset.UtcNow; + var installation = await db.GitHubInstallations.FindAsync([authority.InstallationId], ct).ConfigureAwait(false); + if (installation is not null && installation.ProjectId is not null && + !string.Equals(installation.ProjectId, projectId, StringComparison.Ordinal)) + return RepoAppInstallationBindingOutcome.Conflict; + if (installation is null) + db.GitHubInstallations.Add(new GitHubInstallationRecord + { + InstallationId = authority.InstallationId, AppKind = GitHubAppKind.Repo, ProjectId = projectId, CreatedAt = now, + }); + else + { + installation.ProjectId = projectId; + installation.RevokedAt = null; + } + + var grant = await db.GitHubRepositoryGrants.FindAsync( + [authority.InstallationId, authority.RepositoryId], ct).ConfigureAwait(false); + if (grant is not null && !string.Equals(grant.ProjectId, projectId, StringComparison.Ordinal)) + return RepoAppInstallationBindingOutcome.Conflict; + if (grant is null) + db.GitHubRepositoryGrants.Add(new GitHubRepositoryGrantRecord + { + InstallationId = authority.InstallationId, RepositoryId = authority.RepositoryId, ProjectId = projectId, + FullNameDisplay = authority.FullNameDisplay, + PermissionDigest = RepoAppInstallationTokenService.CreatePermissionDigest(authority.Permissions), + GrantedAt = now, + }); + else + { + var permissionDigest = RepoAppInstallationTokenService.CreatePermissionDigest(authority.Permissions); + if (!CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(grant.PermissionDigest), Encoding.UTF8.GetBytes(permissionDigest))) + { + grant.FullNameDisplay = authority.FullNameDisplay; + grant.RevokedAt = now; + await InvalidateForPermissionChangeAsync(authority.InstallationId, authority.RepositoryId, ct) + .ConfigureAwait(false); + await db.SaveChangesAsync(ct).ConfigureAwait(false); + await transaction.CommitAsync(ct).ConfigureAwait(false); + return RepoAppInstallationBindingOutcome.PermissionChanged; + } + grant.FullNameDisplay = authority.FullNameDisplay; + grant.RevokedAt = null; + } + try + { + await db.SaveChangesAsync(ct).ConfigureAwait(false); + await transaction.CommitAsync(ct).ConfigureAwait(false); + return RepoAppInstallationBindingOutcome.Bound; + } + catch (DbUpdateException) + { + db.ChangeTracker.Clear(); + return RepoAppInstallationBindingOutcome.Conflict; + } + } + + public async Task InvalidateForPermissionChangeAsync( + long installationId, + long repositoryId, + CancellationToken ct = default) + { + var transaction = db.Database.CurrentTransaction is null + ? await db.Database.BeginTransactionAsync(ct).ConfigureAwait(false) + : null; + var now = DateTimeOffset.UtcNow; + try + { + await db.GitHubRepositoryGrants + .Where(x => x.InstallationId == installationId && + x.RepositoryId == repositoryId && + x.RevokedAt == null) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); + await db.AutomationActivations + .Where(x => x.InstallationId == installationId && + x.RepositoryId == repositoryId && + x.Status != AutomationActivationStatus.Invalidated) + .ExecuteUpdateAsync(s => s + .SetProperty(x => x.Status, AutomationActivationStatus.Invalidated) + .SetProperty(x => x.InvalidatedAt, now), ct).ConfigureAwait(false); + if (transaction is not null) + await transaction.CommitAsync(ct).ConfigureAwait(false); + } + finally + { + if (transaction is not null) + await transaction.DisposeAsync().ConfigureAwait(false); + } + } + + private async Task ApplyLifecycleAsync(long installationId, GitHubWebhookPayload payload, CancellationToken ct) + { + var installation = await db.GitHubInstallations.FindAsync([installationId], ct).ConfigureAwait(false); + if (installation is null) + return; // A delivery can never create a project binding from untrusted display data. + + var now = DateTimeOffset.UtcNow; + if (payload.Action is "deleted" or "suspend") + { + installation.RevokedAt = now; + await db.GitHubRepositoryGrants.Where(x => x.InstallationId == installationId && x.RevokedAt == null) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); + return; + } + if (payload.Action is "created" or "unsuspend") + installation.RevokedAt = null; + + foreach (var repository in payload.RepositoriesRemoved ?? []) + { + if (repository.Id > 0) + await db.GitHubRepositoryGrants.Where(x => x.InstallationId == installationId && x.RepositoryId == repository.Id) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); + } + } +} diff --git a/docs/deep-dive/sandboxed-execution.md b/docs/deep-dive/sandboxed-execution.md index 696103473..7e045e2f0 100644 --- a/docs/deep-dive/sandboxed-execution.md +++ b/docs/deep-dive/sandboxed-execution.md @@ -84,6 +84,12 @@ The API (`GET /api/sandbox-policy`, `PUT /api/sandbox-policy`) reads and writes The policy is read through `ISandboxPolicyStore.GetPolicyAsync` and is configurable via the API at `GET /api/sandbox-policy` and `PUT /api/sandbox-policy`. See [sandbox-setup.md](../reference/sandbox-setup.md) for operator instructions. +The API sends one short-lived installation credential for the selected repository and run. The sandbox gives it only to one `git` or `gh` command. + +The approval policy gates `git push`, remote changes, `gh pr` changes, `gh repo` changes, `gh api`, and `gh auth` commands. The API does not inspect or proxy these commands. + +The system keeps this credential out of pod specs, files, logs, events, annotations, shared environments, and credential-helper files. Normal release and orphan cleanup revoke it on a best-effort basis. Token expiry limits a failed revoke. + ## Security model A `run_command` invocation passes three layers before the sandbox engine sees it. diff --git a/packages/Agentweaver.AgentRuntime/CopilotAIAgent.cs b/packages/Agentweaver.AgentRuntime/CopilotAIAgent.cs index 3d6e36d3c..a7245a8c8 100644 --- a/packages/Agentweaver.AgentRuntime/CopilotAIAgent.cs +++ b/packages/Agentweaver.AgentRuntime/CopilotAIAgent.cs @@ -74,6 +74,7 @@ public class CopilotAIAgent : AIAgent, IAsyncDisposable, Workflow.IWorkflowTurnA private readonly IQuestionGate? _questionGate; private readonly IRunOptionsStore? _runOptions; private readonly IEnumerable _toolProviders; + private readonly ISandboxRepositoryCredentialProvider? _repositoryCredentialProvider; // Names of tools built by an IAgentRuntimeToolProvider and wrapped in // InstrumentedCustomAIFunction (populated fresh on every RebuildInnerAgent call). The @@ -263,7 +264,8 @@ public CopilotAIAgent( ILogger logger, IQuestionGate? questionGate = null, IRunOptionsStore? runOptions = null, - IEnumerable? toolProviders = null) + IEnumerable? toolProviders = null, + ISandboxRepositoryCredentialProvider? repositoryCredentialProvider = null) { _factory = factory ?? throw new ArgumentNullException(nameof(factory)); _scopeProvider = scopeProvider ?? throw new ArgumentNullException(nameof(scopeProvider)); @@ -275,6 +277,7 @@ public CopilotAIAgent( _questionGate = questionGate; _runOptions = runOptions; _toolProviders = toolProviders ?? []; + _repositoryCredentialProvider = repositoryCredentialProvider; } /// @@ -406,6 +409,7 @@ public async Task SetupAsync( ? (int)TimeSpan.FromMinutes(10).TotalMilliseconds : (int)TimeSpan.FromMinutes(5).TotalMilliseconds) { + RepositoryAccessToken = _repositoryCredentialProvider?.GetAccessToken(), AllowedRepositoryRoots = [.. sandboxPolicy.AllowedRepositoryRoots], DestructiveCommandPatterns = [.. sandboxPolicy.DestructiveCommandPatterns], RequireApprovalForAllShell = sandboxPolicy.RequireApprovalForAllShell, diff --git a/packages/Agentweaver.AgentTools/ISandboxRepositoryCredentialProvider.cs b/packages/Agentweaver.AgentTools/ISandboxRepositoryCredentialProvider.cs new file mode 100644 index 000000000..7f1f4a18c --- /dev/null +++ b/packages/Agentweaver.AgentTools/ISandboxRepositoryCredentialProvider.cs @@ -0,0 +1,7 @@ +namespace Agentweaver.AgentTools; + +/// Returns the current run's short-lived repository credential. +public interface ISandboxRepositoryCredentialProvider +{ + string? GetAccessToken(); +} diff --git a/packages/Agentweaver.AgentTools/SandboxToolOptions.cs b/packages/Agentweaver.AgentTools/SandboxToolOptions.cs index bb6b95629..3cd4b0a79 100644 --- a/packages/Agentweaver.AgentTools/SandboxToolOptions.cs +++ b/packages/Agentweaver.AgentTools/SandboxToolOptions.cs @@ -7,6 +7,12 @@ public sealed record SandboxToolOptions( bool ShellEnabled, int DefaultTimeoutMs = 300_000) { + /// + /// Short-lived credential for the run's selected repository. The shell tool gives it only to a + /// simple git or gh child process. It is never written to disk or an event. + /// + public string? RepositoryAccessToken { get; init; } + /// /// Allowed repository roots accessible as read-only inside the sandbox. /// If empty, only the working directory is accessible. diff --git a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs index 4960b0726..783c9cee9 100644 --- a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs +++ b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs @@ -99,10 +99,14 @@ public AIFunction CreateFunction(SandboxToolContext ctx) => timeout = ctx.Options.MinimumTimeoutMs; if (ctx.Options.MaximumTimeoutMs > 0) timeout = Math.Min(timeout, ctx.Options.MaximumTimeoutMs); + var environment = BuildCommandEnvironment(ctx.WorkingDirectory, scratchDirectory); + if (!TryAddRepositoryCredential(command, ctx.Options.RepositoryAccessToken, environment, out var credentialError)) + return credentialError!; + var cmd = new SandboxCommand( command, ctx.WorkingDirectory, - BuildCommandEnvironment(ctx.WorkingDirectory, scratchDirectory), + environment, fsPolicy, timeout, NetworkEnabled: ctx.Options.NetworkEnabled, @@ -130,8 +134,8 @@ public AIFunction CreateFunction(SandboxToolContext ctx) => executionLease?.Dispose(); } - var stdout = ctx.Redactor.Redact(result.Stdout); - var stderr = ctx.Redactor.Redact(result.Stderr); + var stdout = RedactOutput(result.Stdout, ctx); + var stderr = RedactOutput(result.Stderr, ctx); var parts = new List(); if (!string.IsNullOrWhiteSpace(stdout)) parts.Add($"stdout:\n{stdout}"); if (!string.IsNullOrWhiteSpace(stderr)) parts.Add($"stderr:\n{stderr}"); @@ -156,7 +160,7 @@ private static string ComputeCommandHash(string command) => ?? Environment.GetEnvironmentVariable("AGENTWEAVER_SCRATCH_DIR"); } - private static IReadOnlyDictionary BuildCommandEnvironment( + private static Dictionary BuildCommandEnvironment( string workingDirectory, string? scratchDirectory) { @@ -187,6 +191,48 @@ private static IReadOnlyDictionary BuildCommandEnvironment( return environment; } + private static bool TryAddRepositoryCredential( + string command, + string? accessToken, + IDictionary environment, + out string? error) + { + error = null; + if (string.IsNullOrWhiteSpace(accessToken)) + return true; + + var trimmed = command.Trim(); + var isGitHubCommand = trimmed.Equals("git", StringComparison.Ordinal) || + trimmed.Equals("gh", StringComparison.Ordinal) || + trimmed.StartsWith("git ", StringComparison.Ordinal) || + trimmed.StartsWith("gh ", StringComparison.Ordinal); + if (!isGitHubCommand) + return true; + + if (trimmed.IndexOfAny([';', '|', '&', '\r', '\n', '`', '<', '>']) >= 0 || + trimmed.Contains("$(", StringComparison.Ordinal)) + { + error = "Command rejected: GitHub credentials require one git or gh command."; + return false; + } + + // GH_TOKEN supports gh and `gh auth git-credential` supports HTTPS Git without a token file. + environment["GH_TOKEN"] = accessToken; + environment["GITHUB_TOKEN"] = accessToken; + environment["GIT_CONFIG_COUNT"] = "1"; + environment["GIT_CONFIG_KEY_0"] = "credential.helper"; + environment["GIT_CONFIG_VALUE_0"] = "!gh auth git-credential"; + return true; + } + + private static string RedactOutput(string value, SandboxToolContext ctx) + { + var redacted = ctx.Redactor.Redact(value); + return string.IsNullOrWhiteSpace(ctx.Options.RepositoryAccessToken) + ? redacted + : redacted.Replace(ctx.Options.RepositoryAccessToken, "***", StringComparison.Ordinal); + } + private static bool IsDestructivePattern(string command, string[] patterns) { if (patterns.Length == 0) return false; diff --git a/packages/Agentweaver.Domain/SandboxPolicy.cs b/packages/Agentweaver.Domain/SandboxPolicy.cs index c39a7829a..423dee792 100644 --- a/packages/Agentweaver.Domain/SandboxPolicy.cs +++ b/packages/Agentweaver.Domain/SandboxPolicy.cs @@ -61,11 +61,15 @@ public sealed record SandboxPolicy "bash <(curl", "sh <(curl", "eval $(curl", "eval $(wget", "| bash", "| sh", // Git destructive - "git push --force", "git push -f", + "git push", "git remote", "git config credential", "git reset --hard", "git push origin --delete", "git push --delete", "git branch -D", "git clean -fd", "git clean -fxd", + // GitHub repository changes and credential commands + "gh pr create", "gh pr merge", "gh pr close", + "gh repo delete", "gh repo archive", + "gh api", "gh auth login", "gh auth logout", // PowerShell destructive "Remove-Item -Recurse", "Remove-Item -Force", "ri -r", "ri -Recurse", "Format-Volume", "Clear-Disk", diff --git a/tests/Agentweaver.Tests/Auth/TwoAppCredentialArchitectureTests.cs b/tests/Agentweaver.Tests/Auth/TwoAppCredentialArchitectureTests.cs index 98a490ca7..2050362cf 100644 --- a/tests/Agentweaver.Tests/Auth/TwoAppCredentialArchitectureTests.cs +++ b/tests/Agentweaver.Tests/Auth/TwoAppCredentialArchitectureTests.cs @@ -44,8 +44,17 @@ public void BrokerSurface_HasOnlyMandatoryPurposeAndOpaqueSnapshotInputs() methods[0].GetParameters().Take(2).Select(parameter => parameter.ParameterType) .Should().Equal(typeof(GitHubCapabilityPurpose), typeof(SnapshotRef)); methods[0].GetParameters().Should().NotContain(parameter => parameter.IsOptional); + + var credential = typeof(GitHubCapabilityBroker).GetMethod( + "TryUseRepositoryCredentialAsync", + BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.DeclaredOnly); + credential.Should().NotBeNull(); + credential.GetParameters().Take(2).Select(parameter => parameter.ParameterType) + .Should().Equal(typeof(SnapshotRef), typeof(DateTimeOffset)); + credential.GetParameters().Should().NotContain(parameter => parameter.IsOptional); File.ReadAllText(Path.Combine(FindRepositoryRoot(), "apps", "Agentweaver.Api", "Auth", "GitHubCapabilityBroker.cs")) - .Should().NotContain("IGitHubTokenScopeProvider").And.NotContain(".Resolve"); + .Should().NotContain("IGitHubTokenScopeProvider").And.NotContain(".Resolve") + .And.NotContain("CommandLine").And.NotContain("git ").And.NotContain("gh "); } [Fact] @@ -106,6 +115,26 @@ public void SnapshotLifecycleIsRunBoundAndDoesNotReachSandboxCredentialDelivery( .Should().Contain("PrepareForLaunchAsync(run, ct)"); } + [Fact] + public void RepositoryCredentialRegistry_UsesOnlyTheRunBoundRepositorySnapshot() + { + var source = File.ReadAllText(Path.Combine( + FindRepositoryRoot(), + "apps", + "Agentweaver.Api", + "Sandbox", + "RunRepositoryCredentialRegistry.cs")); + + source.Should().Contain("GetCapabilitySnapshotsAsync(runId, ct)") + .And.Contain("GitHubCapabilityPurpose.UnattendedRepository") + .And.Contain("TryUseRepositoryCredentialAsync") + .And.Contain("ConcurrentDictionary") + .And.NotContain("CommandLine") + .And.NotContain("endpoint") + .And.NotContain("git ") + .And.NotContain("gh "); + } + [Fact] public void BrowseAuthorityPersistenceRemainsOwnedByProjectCreationFlow() { diff --git a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs index 258f976c1..d52fff9c6 100644 --- a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs @@ -99,11 +99,25 @@ public void Assembly_session_registers_controlled_run_command_custom_tool() [InlineData("nohup npm test")] [InlineData("setsid npm test")] [InlineData("rm -rf node_modules")] + [InlineData("git push --force origin main")] + [InlineData("git remote set-url origin https://github.com/example/repo")] + [InlineData("git config credential.helper store")] + [InlineData("gh pr create --title test --body test")] + [InlineData("gh pr merge 1")] + [InlineData("gh pr close 1")] + [InlineData("gh repo delete example/repo")] + [InlineData("gh repo archive example/repo")] + [InlineData("gh api /user")] + [InlineData("gh auth login")] + [InlineData("gh auth logout")] public async Task Controlled_run_command_rejects_backgrounding_and_destructive_commands(string command) { var executor = new CountingExecutor(); using var tracker = new ShellExecutionTracker(); - var context = BuildContext(executor, tracker); + var context = BuildContext( + executor, + tracker, + destructivePatterns: [.. SandboxPolicy.Default(_root).DestructiveCommandPatterns]); var tool = CopilotAIAgent.BuildSessionConfigTools( context, includeControlledRunCommand: true).Single(t => t.Name == "run_command"); @@ -115,6 +129,84 @@ public async Task Controlled_run_command_rejects_backgrounding_and_destructive_c executor.ExecuteCalls.Should().Be(0); } + [Theory] + [InlineData("git status")] + [InlineData("gh repo view")] + public async Task Controlled_run_command_supplies_repository_credential_only_to_git_and_gh(string command) + { + SandboxCommand? observed = null; + var executor = new CapturingExecutor(command => observed = command); + using var tracker = new ShellExecutionTracker(); + var tool = CopilotAIAgent.BuildSessionConfigTools( + BuildContext(executor, tracker, repositoryAccessToken: "repository-access-token"), + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = command })); + + observed.Should().NotBeNull(); + observed!.Environment.Should().Contain( + new KeyValuePair("GH_TOKEN", "repository-access-token")); + observed.Environment.Should().Contain( + new KeyValuePair("GITHUB_TOKEN", "repository-access-token")); + observed.Environment.Should().Contain( + new KeyValuePair("GIT_CONFIG_VALUE_0", "!gh auth git-credential")); + observed.CommandLine.Should().NotContain("repository-access-token"); + } + + [Fact] + public async Task Controlled_run_command_does_not_supply_repository_credential_to_other_commands() + { + SandboxCommand? observed = null; + var executor = new CapturingExecutor(command => observed = command); + using var tracker = new ShellExecutionTracker(); + var tool = CopilotAIAgent.BuildSessionConfigTools( + BuildContext(executor, tracker, repositoryAccessToken: "repository-access-token"), + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = "dotnet --info" })); + + observed.Should().NotBeNull(); + observed!.Environment.Should().NotContain( + pair => pair.Value == "repository-access-token"); + } + + [Fact] + public async Task Controlled_run_command_redacts_repository_credential_from_command_output() + { + var executor = new CapturingExecutor(_ => { }, "repository-access-token"); + using var tracker = new ShellExecutionTracker(); + var tool = CopilotAIAgent.BuildSessionConfigTools( + BuildContext(executor, tracker, repositoryAccessToken: "repository-access-token"), + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var result = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = "git status" })); + + result?.ToString().Should().Contain("***").And.NotContain("repository-access-token"); + } + + [Theory] + [InlineData("git status; whoami")] + [InlineData("gh repo view > output.txt")] + [InlineData("git $(echo status)")] + [InlineData("gh repo view\r\nwhoami")] + public async Task Controlled_run_command_rejects_compound_credentialed_commands(string command) + { + var executor = new CountingExecutor(); + using var tracker = new ShellExecutionTracker(); + var tool = CopilotAIAgent.BuildSessionConfigTools( + BuildContext(executor, tracker, repositoryAccessToken: "repository-access-token"), + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var result = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = command })); + + result?.ToString().Should().Contain("GitHub credentials require one git or gh command"); + executor.ExecuteCalls.Should().Be(0); + } + [Fact] public async Task Controlled_run_command_arms_watchdog_deadline_above_executor_timeout_by_grace() { @@ -364,7 +456,9 @@ private SandboxToolContext BuildContext( ISandboxExecutor executor, ShellExecutionTracker tracker, string? workspace = null, - string? scratchDirectory = null) => + string? scratchDirectory = null, + string? repositoryAccessToken = null, + string[]? destructivePatterns = null) => new( AgentId: "agent", WorkingDirectory: workspace ?? _root, @@ -375,10 +469,11 @@ private SandboxToolContext BuildContext( Redactor: SandboxOutputRedactor.Default, Options: new SandboxToolOptions(ShellEnabled: true, DefaultTimeoutMs: 600_000) { - DestructiveCommandPatterns = ["rm -rf"], + DestructiveCommandPatterns = destructivePatterns ?? ["rm -rf"], RejectBackgroundCommands = true, RejectDestructiveCommands = true, MaximumTimeoutMs = 600_000, + RepositoryAccessToken = repositoryAccessToken, }, Logger: NullLogger.Instance, ShellExecutionTracker: tracker, @@ -410,7 +505,7 @@ public IAsyncEnumerable StreamAsync( } } - private sealed class CapturingExecutor(Action onExecute) : ISandboxExecutor + private sealed class CapturingExecutor(Action onExecute, string stdout = "ok") : ISandboxExecutor { public bool IsRealIsolation => false; public string BackendName => "direct"; @@ -424,7 +519,7 @@ public Task ExecuteAsync( { onExecute(command); return Task.FromResult( - new SandboxExecResult(0, "ok", "", TimedOut: false, OutputTruncated: false)); + new SandboxExecResult(0, stdout, "", TimedOut: false, OutputTruncated: false)); } public async IAsyncEnumerable StreamAsync( From 9661bc4f8681eeba27eabb7c53b2fee4db97062d Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 20:23:19 -0700 Subject: [PATCH 02/12] chore: preserve source line endings Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- .../Sandbox/KubernetesSandboxExecutor.cs | 3078 ++++++++--------- .../Sandbox/SandboxExecutorRouter.cs | 368 +- .../Webhooks/RepoAppInstallationService.cs | 1142 +++--- 3 files changed, 2294 insertions(+), 2294 deletions(-) diff --git a/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs b/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs index 4c2425901..e858b33bb 100644 --- a/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs +++ b/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs @@ -1,1539 +1,1539 @@ -using System.Runtime.CompilerServices; -using System.Security.Cryptography; -using System.Text; -using System.Net.Http.Json; -using System.Net.Sockets; -using System.Text.Json; -using Agentweaver.Api.Auth; -using Agentweaver.Api.Infrastructure; -using Agentweaver.AgentRuntime.Workflow; -using Agentweaver.Domain; -using k8s; -using k8s.Autorest; -using Agentweaver.SandboxExec; -using Microsoft.Extensions.Logging; - -namespace Agentweaver.Api.Sandbox; - -/// -/// Configures the Kubernetes SandboxClaim backend. -/// Bound from the Sandbox:Kubernetes configuration section. -/// -public sealed class KubernetesSandboxOptions -{ - public string Namespace { get; init; } = "agentweaver"; - public string TemplateRef { get; init; } = "agentweaver-sandbox"; - /// - /// SandboxWarmPool the generic command-exec claim binds to. In the v1beta1 CRD a - /// SandboxClaim references a SandboxWarmPool (spec.warmPoolRef.name), - /// which in turn references the SandboxTemplate. Default: agentweaver-sandbox. - /// - public string WarmPoolRef { get; init; } = "agentweaver-sandbox"; - /// Path where the shared workspace PVC is mounted inside API and sandbox pods. - public string WorkspaceMountPath { get; init; } = "/workspace"; - /// SandboxClaim TTL. Command timeouts are capped below this so controller GC cannot interrupt exec. - public int TimeoutSeconds { get; init; } = 600; - /// Cluster service CIDR that must be excluded by sandbox egress policy. - public string? ServiceCidr { get; init; } - public IReadOnlyList SandboxEgressCidrExclusions { get; init; } = []; - - // ── Pod-per-run AgentHost lifecycle options (spec §9 / Q3 hybrid) ───────── - - /// - /// SandboxWarmPool the AgentHost (pod-per-run) claim binds to in the v0.5.0 v1beta1 CRD - /// (spec.warmPoolRef.name). The pool itself references the AgentHost SandboxTemplate. - /// Default: agentweaver-agent-host. - /// - public string AgentHostWarmPoolRef { get; init; } = "agentweaver-agent-host"; - - /// - /// Port the AgentHost Kestrel listener binds to inside the pod. - /// Worker builds the A2A endpoint as http://<podIP>:<AgentHostPort><AgentHostA2APath>. - /// TLS/mTLS termination is owned by Link (H1) — leave hook here for cert wiring. - /// Default: 8088. - /// - public int AgentHostPort { get; init; } = 8088; - - /// - /// A2A path prefix mounted by MapA2AHttpJson inside the AgentHost pod. - /// Must match AgentHost:A2APath set in the pod's configuration. - /// Default: /a2a/agent. - /// - public string AgentHostA2APath { get; init; } = "/a2a/agent"; - - /// - /// When (default) the AgentHost A2A endpoint uses https with - /// mTLS (H1). When (PoC only) it uses plain http. Drives the - /// scheme via and is injected into the pod as - /// AgentHost__RequireMtls. Config key: Sandbox:AgentHost:RequireMtls. - /// - public bool RequireMtls { get; init; } = true; - - // ── AgentHost readiness gate (A2A cold-start race) ─────────────────────── - - /// - /// Path the AgentHost exposes for liveness/readiness on . The executor - /// polls {scheme}://{podIP}:{port}{AgentHostHealthzPath} after the claim binds and BEFORE - /// returning the A2A endpoint, so the worker never sends the first turn into the Kestrel boot - /// window (which would be refused). Default: /healthz. - /// - public string AgentHostHealthzPath { get; init; } = "/healthz"; - - /// - /// Maximum time to wait for the AgentHost to start serving - /// before failing the launch deterministically. Default: 90s (covers cold-start Kestrel bind). - /// - public int AgentHostReadyTimeoutSeconds { get; init; } = 90; - - /// Interval between AgentHost readiness probe attempts. Default: 1000ms. - public int AgentHostReadyPollIntervalMs { get; init; } = 1000; - - /// - /// Minimum age before the orphan reaper may delete an AgentHost claim that is absent from the - /// active-run map. Config key: Sandbox:Kubernetes:AgentHostClaimCreationGraceSeconds. - /// The effective value is floored above . - /// Default: 300s. - /// - public int AgentHostClaimCreationGraceSeconds { get; init; } = 300; - - /// - /// Azure Key Vault URI injected into AgentHost pods as AgentHost__KeyVaultUri so the - /// warm pod can fetch the run owner's GitHub token via workload identity at /configure-time - /// (Option C). Sourced from the API's own KV config (Auth:TokenStore:KeyVaultUri). When - /// null/empty the env var is omitted and the pod falls back to the CSI file-mount path. - /// - public string? KvUri { get; init; } -} - -/// -/// Top-level sandbox runtime options bound from the Sandbox configuration section -/// (not under Sandbox:Kubernetes). Controls the agent-execution mode and -/// the pod-release-on-suspend behaviour (Q3 hybrid). -/// -public sealed class SandboxRuntimeOptions -{ - /// - /// Agent execution mode. - /// - /// in-api (default) — run agents in-process; instant rollback path (§4.7.6). - /// pod-per-run — launch a per-run AgentHost sandbox pod; activate A2A transport. - /// - /// - public string AgentExecutionMode { get; init; } = "in-api"; - - /// - /// When true (default) and is pod-per-run, - /// the AgentHost pod is released (SandboxClaim deleted) whenever the MAF graph suspends - /// at a RequestPort (HITL/review gate) or the coordinator idles awaiting children. - /// Set to false to keep the pod warm across suspension (lower resume latency, higher - /// resource cost; recommended only for short-wait HITL in dev/staging). - /// - public bool ReleasePodOnSuspend { get; init; } = true; - - /// - public bool IsPodPerRun => - string.Equals(AgentExecutionMode, "pod-per-run", StringComparison.OrdinalIgnoreCase); -} - -/// -/// Executes sandboxed commands inside a pre-warmed Kubernetes pod obtained via a -/// SandboxClaim CRD. Lifecycle: -/// -/// Create a SandboxClaim resource (adopts a warm pod from the pool). -/// Poll until the claim transitions to phase: Bound and reports a pod name. -/// Run the command via pod-exec (Kubernetes WebSocket exec API). -/// Delete the claim on completion (controller GC cleans up the pod and service). -/// -/// Automatically selected by the API when KUBERNETES_SERVICE_HOST is present -/// (see ). -/// -internal sealed class KubernetesSandboxExecutor : ISandboxExecutor, IAgentHostPodLifecycle -{ - private const string ApiGroup = SandboxClaimConventions.ApiGroup; - private const string ApiVersion = SandboxClaimConventions.ApiVersion; - private const string ClaimPlural = SandboxClaimConventions.ClaimPlural; - private const string ContainerName = "agentweaver-sandbox"; - - /// - /// Bounded attempt count for — the total number of - /// tries (initial + retries) for a transient Kubernetes API fault (issue #230). A transient - /// connection reset (SocketException 104 → IOException → HttpRequestException) that used to fail - /// a subtask fatally is now retried with exponential backoff + jitter. - /// - private const int MaxK8sAttempts = 3; - - /// - /// Cadence for the heartbeat emitted while an - /// AgentHost SandboxClaim is still being provisioned (unbound). Must stay well under the - /// parent coordinator's Coordinator:SubtaskStallTimeoutMinutes (default 5 min) so each - /// provisioning wait window is punctuated by an event that keeps the outbound stream flowing and - /// resets the stall timer (issue #217, mirrors the #212 tool.approval_pending heartbeat cadence). - /// - internal static readonly TimeSpan SandboxProvisioningHeartbeatInterval = TimeSpan.FromSeconds(20); - - private readonly IKubernetes _client; - private readonly KubernetesSandboxOptions _options; - private readonly ILogger _logger; - private readonly IPodNameRegistry? _podRegistry; - private readonly IAgentHostTurnTokenRegistry? _turnTokenRegistry; - private readonly Security.IRunAuthorshipCapabilityStore? _authorshipCapabilityStore; - // Polls the AgentHost /healthz after bind and before returning the endpoint, closing the - // A2A cold-start race (pod Running ~20-30s before Kestrel binds :8088). Null in unit tests - // that only assert the claim body → readiness gate is skipped. - private readonly IAgentHostReadinessProbe? _readinessProbe; - // Resolves the run's submitting user so the pod can be scoped (via /configure) to the run owner's - // Copilot-entitled token instead of the installation token. Null when the run→user lookup is - // unavailable. - private readonly IRunSubmittingUserResolver? _submittingUserResolver; - // Used to POST /configure to the warm pod after bind (warm-pool deferred-config path). Null in - // unit tests → the /configure call is skipped (same null-skip convention as the readiness probe). - private readonly IHttpClientFactory? _httpClientFactory; - // Resolves the run owner's GitHub token so the API can pass it in /configure, avoiding the need - // for the kata VM pod to call Azure AD or Key Vault (blocked by Cilium FQDN policies). - private readonly IGitHubTokenStore? _tokenStore; - private readonly IGitHubTokenScopeProvider? _tokenScopeProvider; - // Refresh-aware token accessor (issue #523): a Build & Test gate can launch its AgentHost pod for - // the FIRST time (a fresh pod, not yet /configure'd for this run) many minutes after the run's - // earlier subtask stages — long enough for the submitting user's Copilot-entitled OAuth access - // token to cross its expiry skew window. Reading the raw entry via IGitHubTokenStore.GetAsync (as - // ResolveGitHubAccessTokenAsync previously did) can hand a stale/expired access token to the pod, - // which the pod then trusts unconditionally (its "fast path" skips its own Key Vault fetch - // whenever a pre-resolved token arrives) — producing GitHubCopilotUnauthorizedException at - // /configure. Routing through the same GetValidAccessTokenAsync used by GitHubCopilotClientFactory - // ensures a near-expiry token is transparently rotated before being handed to a newly-launched pod. - // Null in unit tests → falls back to the raw (non-refreshing) token store read. When present, - // it is authoritative: a null/failed refresh must never fall back to the rejected raw token. - private readonly IGitHubAccessTokenProvider? _accessTokenProvider; - // Replica-safe run secret store used to persist the per-run preview-runner credential so a - // reconcile/keepalive on either API replica can re-fetch it, and to durably DELETE it on pod - // release (spec-006 decouple-preview, BLOCKER A / RESIDUAL). Null in unit tests → no minting. - private readonly ISecretStore? _secretStore; - // Durable run-event log used to emit sandbox.provisioning_pending heartbeats into the CHILD run's - // stream while its AgentHost claim is still being scheduled by Kubernetes (unbound). Keeps the - // parent coordinator's stall timer alive during a legitimately-long Pending wait (issue #217). - // Null in unit tests → the heartbeat is skipped (same null-skip convention as the readiness probe). - private readonly IRunEventStream? _runEventStream; - // Source of the per-run AutoApproveTools flag propagated to the warm pod via /configure (bug - // #221). Null in unit tests → the flag defaults false (same null-skip convention as above). - private readonly IRunOptionsStore? _runOptions; - private readonly RunRepositoryCredentialRegistry? _repositoryCredentials; - // First-class preview lifecycle reconciler. ReleaseAgentHostPodAsync derives durable - // Previewable/PreviewActive state and applies all retention or cleanup effects before deciding - // whether to delete the claim. - private readonly Agentweaver.Api.Sandbox.Preview.ISandboxPreviewService? _previewService; - - public bool IsRealIsolation => true; - public string BackendName => "kubernetes-sandbox-claim"; - public string SelectionReason => - "Kubernetes-native sandbox via SandboxClaim warm pool (Kata VM isolation, NetworkPolicy egress restriction)."; - public bool HasNetworkWarning => false; - public string? NetworkWarningMessage => null; - - internal KubernetesSandboxExecutor( - IKubernetes client, - KubernetesSandboxOptions options, - ILogger logger, - IPodNameRegistry? podRegistry = null, - IAgentHostTurnTokenRegistry? turnTokenRegistry = null, - IAgentHostReadinessProbe? readinessProbe = null, - IRunSubmittingUserResolver? submittingUserResolver = null, - IHttpClientFactory? httpClientFactory = null, - IGitHubTokenStore? tokenStore = null, - ISecretStore? secretStore = null, - IRunEventStream? runEventStream = null, - IRunOptionsStore? runOptions = null, - RunRepositoryCredentialRegistry? repositoryCredentials = null, - IGitHubAccessTokenProvider? accessTokenProvider = null, - Agentweaver.Api.Sandbox.Preview.ISandboxPreviewService? previewService = null, - IGitHubTokenScopeProvider? tokenScopeProvider = null, - Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null) - { - _client = client; - _options = options; - _logger = logger; - _podRegistry = podRegistry; - _turnTokenRegistry = turnTokenRegistry; - _readinessProbe = readinessProbe; - _submittingUserResolver = submittingUserResolver; - _httpClientFactory = httpClientFactory; - _tokenStore = tokenStore; - _tokenScopeProvider = tokenScopeProvider; - _secretStore = secretStore; - _runEventStream = runEventStream; - _runOptions = runOptions; - _repositoryCredentials = repositoryCredentials; - _accessTokenProvider = accessTokenProvider; - _previewService = previewService; - _authorshipCapabilityStore = authorshipCapabilityStore; - } - - public async Task ExecuteAsync( - SandboxCommand command, CancellationToken ct = default) - { - // Use the Agentweaver run ID as the claim name when available so the pod can be - // looked up by run ID later (preview port-forward). Fall back to a random ID. - var claimName = string.IsNullOrEmpty(command.AgentweaverRunId) - ? $"run-{Guid.NewGuid():N}"[..20] - : SandboxClaimConventions.DeriveRunCommandClaimName(command.AgentweaverRunId); - - var requestedTimeoutMs = command.TimeoutMs > 0 - ? command.TimeoutMs - : _options.TimeoutSeconds * 1000; - var maxCommandTimeoutMs = Math.Max(1000, (_options.TimeoutSeconds * 1000) - 30_000); - var timeoutMs = Math.Min(requestedTimeoutMs, maxCommandTimeoutMs); - if (timeoutMs < requestedTimeoutMs) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: command timeout clamped from {RequestedMs}ms to {TimeoutMs}ms so it stays below SandboxClaim TTL ({TtlSeconds}s)", - requestedTimeoutMs, timeoutMs, _options.TimeoutSeconds); - } - - string podWorkingDirectory; - try - { - podWorkingDirectory = ResolvePodWorkingDirectory(command.WorkingDirectory); - } - catch (Exception ex) - { - _logger.LogError(ex, - "KubernetesSandboxExecutor: invalid workspace path {WorkingDirectory}; configured mount is {WorkspaceMountPath}", - command.WorkingDirectory, _options.WorkspaceMountPath); - return new SandboxExecResult(1, "", ex.Message, false, false); - } - - _logger.LogInformation( - "KubernetesSandboxExecutor: using workspace path {WorkspacePath} for claim {Claim} (requested {RequestedWorkingDirectory})", - podWorkingDirectory, claimName, command.WorkingDirectory); - - using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct); - linked.CancelAfter(timeoutMs); - var token = linked.Token; - var claimCreated = false; - - try - { - _logger.LogInformation( - "KubernetesSandboxExecutor: creating SandboxClaim {Claim}", claimName); - claimCreated = await CreateClaimAsync(claimName, token); - - var podName = await WaitForBoundAsync(claimName, token); - _logger.LogInformation( - "KubernetesSandboxExecutor: claim {Claim} bound to pod {Pod}", claimName, podName); - - // Register pod name so PortForwardService can locate it by Agentweaver run ID. - // Run-scoped mappings are cleared by run lifecycle cleanup, not per command, so - // preview tunnels can remain available for the whole run while the claim TTL is valid. - if (!string.IsNullOrEmpty(command.AgentweaverRunId)) - _podRegistry?.Register(command.AgentweaverRunId, podName); - - return await ExecInPodAsync(podName, command, podWorkingDirectory, token); - } - catch (OperationCanceledException) when (!ct.IsCancellationRequested) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: timed out waiting for claim {Claim}", claimName); - return new SandboxExecResult(-1, "", "Timed out waiting for sandbox pod.", true, false); - } - finally - { - if (claimCreated && string.IsNullOrEmpty(command.AgentweaverRunId)) - await DeleteClaimAsync(claimName); - else if (claimCreated) - _logger.LogDebug( - "KubernetesSandboxExecutor: retaining SandboxClaim {Claim} for run {RunId} preview until run cleanup or TTL", - claimName, command.AgentweaverRunId); - } - } - - public async IAsyncEnumerable StreamAsync( - SandboxCommand command, - [EnumeratorCancellation] CancellationToken ct = default) - { - var result = await ExecuteAsync(command, ct); - foreach (var line in result.Stdout.Split('\n')) - yield return new SandboxOutputChunk(SandboxOutputStream.Stdout, line); - if (!string.IsNullOrEmpty(result.Stderr)) - foreach (var line in result.Stderr.Split('\n')) - yield return new SandboxOutputChunk(SandboxOutputStream.Stderr, line); - yield return new SandboxOutputChunk(SandboxOutputStream.ExitCode, result.ExitCode.ToString()); - } - - // ── IAgentHostPodLifecycle — pod-per-run lifecycle (spec §9 / Q3) ───────────── - - /// - public Task LaunchAgentHostPodAsync(string runId, CancellationToken ct = default) => - LaunchAgentHostPodAsync(runId, new AgentHostLaunchContext(SharedWorkingDirectory: null), ct); - - /// - public Task LaunchAgentHostPodAsync( - string runId, - string? workingDirectoryOverride, - CancellationToken ct = default) => - LaunchAgentHostPodAsync( - runId, - new AgentHostLaunchContext(SharedWorkingDirectory: workingDirectoryOverride), - ct); - - /// - public async Task LaunchAgentHostPodAsync( - string runId, - AgentHostLaunchContext launchContext, - CancellationToken ct = default) - { - var claimName = SandboxClaimConventions.DeriveAgentHostClaimName(runId); - var requestedWorkingDirectory = string.IsNullOrWhiteSpace(launchContext.SharedWorkingDirectory) - ? null - : Path.GetFullPath(launchContext.SharedWorkingDirectory); - - _logger.LogInformation( - "KubernetesSandboxExecutor: launching AgentHost pod for run {RunId} via claim {Claim}", - runId, claimName); - - // Resolve the run's submitting user so the pod can scope GitHub Copilot auth to that user's - // signed-in token. The user's Key Vault secret name (Option C warm-pool path) is derived here - // and delivered to the pod via /configure — never another user's secret. - var submittingUser = await ResolveSubmittingUserAsync(runId, ct).ConfigureAwait(false); - if (string.IsNullOrWhiteSpace(submittingUser)) - { - throw new InvalidOperationException( - $"Cannot launch AgentHost pod for run '{runId}' without a submitting user; " + - "the /configure call must scope the pod to the run owner's Key Vault token."); - } - - _logger.LogInformation( - "KubernetesSandboxExecutor: resolved submitting user for run {RunId}; will configure pod via /configure.", - runId); - - var (configProjectId, configAgentName) = _submittingUserResolver is not null - ? await _submittingUserResolver.GetRunIdentityAsync(runId, ct).ConfigureAwait(false) - : (null, null); - - // ghtok-user--{base32(userId)} — the SAME mapping the API uses when persisting the token to KV. - // With Entra sign-in the user's credentials live under the ACTIVE linked GitHub identity's - // scope (user-link:{oid}:{login}), so resolve the effective scope rather than assuming the - // legacy per-user scope, which is never written in that mode. - var effectiveScope = _tokenScopeProvider is not null - ? await _tokenScopeProvider - .ResolveAsync(submittingUser!, configProjectId, ct) - .ConfigureAwait(false) - : _tokenStore is IEffectiveGitHubTokenScopeResolver scopeResolver - ? await scopeResolver.ResolveEffectiveScopeAsync(submittingUser!, ct).ConfigureAwait(false) - : GitHubTokenScope.ForUser(submittingUser!); - var kvUserSecretName = KeyVaultSecretStore.SanitizeKey(effectiveScope.Key); - var turnToken = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)); - var claimCreated = false; - try - { - // Bind to the SHARED, pre-warmed AgentHost warm pool (replicas: 2). No per-run SPC, - // SandboxTemplate, or warm pool — the pod is already warm and gets its per-run context - // via the /configure POST below. - claimCreated = await CreateAgentHostClaimAsync( - claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); - - if (!claimCreated && launchContext.Purpose == AgentHostPurpose.OperatorAssistant) - { - // Every operator turn carries the CURRENT browser/platform bearer. An orphaned - // claim from a crashed prior turn is already configured with the old credential - // and /configure is intentionally one-shot, so it must never be reused. - _logger.LogInformation( - "KubernetesSandboxExecutor: recreating existing AgentHost claim {Claim} for a fresh operator-assistant caller credential.", - claimName); - await DeleteClaimAsync(claimName).ConfigureAwait(false); - _podRegistry?.Unregister(runId); - _turnTokenRegistry?.UnregisterTurnToken(runId); - await Task.Delay(1000, ct).ConfigureAwait(false); - claimCreated = await CreateAgentHostClaimAsync( - claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); - if (!claimCreated) - { - throw new InvalidOperationException( - $"AgentHost claim '{claimName}' was deleted to refresh the operator-assistant caller credential, " + - "but the replacement create still conflicted."); - } - } - else if (!claimCreated && launchContext.WorkspaceMode != ExecutionWorkspaceMode.Shared) - { - _logger.LogInformation( - "KubernetesSandboxExecutor: recreating existing AgentHost claim {Claim} for immutable pod-local workspace configuration (mode={Mode}).", - claimName, - launchContext.WorkspaceMode); - await DeleteClaimAsync(claimName).ConfigureAwait(false); - _podRegistry?.Unregister(runId); - _turnTokenRegistry?.UnregisterTurnToken(runId); - await Task.Delay(1000, ct).ConfigureAwait(false); - claimCreated = await CreateAgentHostClaimAsync( - claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); - if (!claimCreated) - { - throw new InvalidOperationException( - $"AgentHost claim '{claimName}' was deleted for immutable pod-local workspace configuration, " + - "but the replacement create still conflicted."); - } - } - else if (!claimCreated && requestedWorkingDirectory is not null) - { - var existingWorkingDirectory = await TryGetAgentHostClaimWorkingDirectoryAsync(claimName, ct) - .ConfigureAwait(false); - var sameWorktree = string.Equals( - existingWorkingDirectory, requestedWorkingDirectory, StringComparison.Ordinal); - var hasTurnToken = !string.IsNullOrWhiteSpace(_turnTokenRegistry?.TryGetTurnToken(runId)); - - if (!sameWorktree || !hasTurnToken) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: existing AgentHost claim {Claim} for run {RunId} " + - "is not reusable (sameWorktree={SameWorktree}, hasTurnToken={HasTurnToken}); recreating.", - claimName, runId, sameWorktree, hasTurnToken); - await DeleteClaimAsync(claimName).ConfigureAwait(false); - _podRegistry?.Unregister(runId); - _turnTokenRegistry?.UnregisterTurnToken(runId); - await Task.Delay(1000, ct).ConfigureAwait(false); - claimCreated = await CreateAgentHostClaimAsync( - claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); - if (!claimCreated) - { - throw new InvalidOperationException( - $"AgentHost claim '{claimName}' for run '{runId}' was deleted for worktree reconfiguration, " + - "but the replacement create still conflicted. Retrying later avoids reusing a token-less or stale pod."); - } - } - } - - var podName = await WaitForBoundWithProvisioningHeartbeatAsync(runId, claimName, ct).ConfigureAwait(false); - _logger.LogInformation( - "KubernetesSandboxExecutor: AgentHost claim {Claim} bound to pod {Pod}", claimName, podName); - - // Register also persists sandbox.execution_pod.bound into the shared RunEvents store so - // graph snapshots/deltas on any API replica can resolve the execution pod. - _podRegistry?.Register(runId, podName); - if (claimCreated) - _turnTokenRegistry?.RegisterTurnToken(runId, turnToken); - - var activeTurnToken = claimCreated - ? turnToken - : _turnTokenRegistry?.TryGetTurnToken(runId); - if (_authorshipCapabilityStore is not null && !string.IsNullOrWhiteSpace(activeTurnToken)) - { - await _authorshipCapabilityStore.RegisterAsync( - runId, activeTurnToken, DateTimeOffset.UtcNow.AddDays(1), ct).ConfigureAwait(false); - } - - var podIp = await GetPodIpAsync(podName, ct).ConfigureAwait(false); - - var endpointUrl = AgentHostEndpoint.Build( - _options.RequireMtls, podIp, _options.AgentHostPort, _options.AgentHostA2APath); - - // A2A cold-start gate: the claim binds when the pod is Running, but the AgentHost Kestrel - // listener takes ~20-30s more to bind :8088. Without this wait the worker's first A2A POST - // hits a closed port → "Connection refused" → the run fails mid-turn. Poll /healthz until the - // app is actually serving so a not-yet-ready pod is a deterministic LAUNCH failure instead. - // NOTE: a warm/standby pod serves /healthz BEFORE /configure (the readiness gate exempts - // /configure), so this confirms reachability prior to injecting the run context. - if (_readinessProbe is not null) - { - var scheme = AgentHostEndpoint.Scheme(_options.RequireMtls); - var readinessUrl = - $"{scheme}://{podIp}:{_options.AgentHostPort}{_options.AgentHostHealthzPath}"; - - _logger.LogInformation( - "KubernetesSandboxExecutor: waiting for AgentHost readiness for run {RunId} at {Url}", - runId, readinessUrl); - - try - { - await _readinessProbe.WaitUntilReadyAsync(readinessUrl, ct).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - throw new InvalidOperationException( - $"AgentHost pod '{podName}' for run '{runId}' did not become ready at {readinessUrl} " + - $"within {_options.AgentHostReadyTimeoutSeconds}s; failing the launch.", ex); - } - } - - // Warm-pool deferred /configure: inject the per-run RunId/UserId/TurnBearerToken and the - // KV secret name into the already-warm pod, which then runs SetupAsync and becomes ready. - // Normal roles use the shared orchestration worktree. Local workspace modes carry - // immutable source refs; AgentHost creates their effective root inside execution-scratch. - if (claimCreated) - { - var repositoryAccessToken = _repositoryCredentials is null - ? null - : await _repositoryCredentials.MintAsync(runId, ct).ConfigureAwait(false); - var effectiveWorkingDirectory = await CallAgentHostConfigureAsync( - podIp, _options.AgentHostPort, runId, submittingUser, turnToken, kvUserSecretName, - effectiveScope, - await ResolveGitHubAccessTokenAsync(effectiveScope, submittingUser, ct).ConfigureAwait(false), - repositoryAccessToken, - requestedWorkingDirectory ?? await ResolveWorkingDirectoryAsync(runId, ct).ConfigureAwait(false), - launchContext, - configProjectId, - configAgentName, - ct) - .ConfigureAwait(false); - if (!string.IsNullOrWhiteSpace(effectiveWorkingDirectory)) - _podRegistry?.RegisterEffectiveWorkingDirectory(runId, effectiveWorkingDirectory); - } - else - { - _logger.LogInformation( - "KubernetesSandboxExecutor: reusing already-configured AgentHost claim {Claim} for run {RunId}", - claimName, runId); - } - - _podRegistry?.RegisterAgentEndpoint(runId, endpointUrl); - - _logger.LogInformation( - "KubernetesSandboxExecutor: AgentHost A2A endpoint for run {RunId} = {Endpoint}", - runId, endpointUrl); - - return endpointUrl; - } - catch - { - if (claimCreated) - await DeleteClaimAsync(claimName).ConfigureAwait(false); - _podRegistry?.Unregister(runId); - _turnTokenRegistry?.UnregisterTurnToken(runId); - if (_authorshipCapabilityStore is not null) - { - await _authorshipCapabilityStore.RemoveAsync(runId, CancellationToken.None) - .ConfigureAwait(false); - } - // Crash/timeout during launch: delete any credential minted before the failure so it is - // never left behind (spec-006 decouple-preview, RESIDUAL rev3 gap). - await DeletePreviewRunnerCredentialAsync(runId, CancellationToken.None).ConfigureAwait(false); - await RevokeRepositoryCredentialAsync(runId, CancellationToken.None).ConfigureAwait(false); - throw; - } - } - - /// - public async Task ReleaseAgentHostPodAsync(string runId, CancellationToken ct = default) - { - var claimName = SandboxClaimConventions.DeriveAgentHostClaimName(runId); - - // Issue #542: if a live preview is still active for this run, releasing the pod here (at the - // originating subtask's turn end) would 404 the preview URL before any human-review gate or - // demo viewer can open it. Defer the claim delete while the preview is alive; the preview's own - // idle/max expiry + the reaper will eventually reap the pod, so this cannot leak. - if (_previewService is not null && - await _previewService.ReconcilePreviewLifecycleAsync(runId, ct).ConfigureAwait(false) - == Agentweaver.Api.Sandbox.Preview.PreviewLifecycleState.PreviewActive) - { - _logger.LogInformation( - "KubernetesSandboxExecutor: deferring AgentHost pod release for run {RunId} (claim " + - "{Claim}) — a live preview is still active; the preview idle/max expiry will reap it.", - runId, claimName); - return; - } - - _logger.LogInformation( - "KubernetesSandboxExecutor: releasing AgentHost pod for run {RunId} (claim {Claim})", - runId, claimName); - - await DeleteClaimAsync(claimName, ct).ConfigureAwait(false); - _podRegistry?.Unregister(runId); - _turnTokenRegistry?.UnregisterTurnToken(runId); - if (_authorshipCapabilityStore is not null) - await _authorshipCapabilityStore.RemoveAsync(runId, ct).ConfigureAwait(false); - await DeletePreviewRunnerCredentialAsync(runId, ct).ConfigureAwait(false); - await RevokeRepositoryCredentialAsync(runId, ct).ConfigureAwait(false); - - _logger.LogInformation( - "KubernetesSandboxExecutor: AgentHost pod released for run {RunId}", runId); - } - - /// - /// Resolves the submitting user for via the injected resolver, never - /// throwing (a lookup failure must not fail the launch — it degrades to omitting the user id). - /// - private async Task ResolveSubmittingUserAsync(string runId, CancellationToken ct) - { - if (_submittingUserResolver is null) - return null; - - try - { - return await _submittingUserResolver.GetSubmittingUserAsync(runId, ct).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger.LogWarning( - ex, - "KubernetesSandboxExecutor: failed to resolve submitting user for run {RunId}; " + - "AgentHost__UserId will be omitted.", - runId); - return null; - } - } - - /// - /// Resolves the per-run working directory (shared orchestration worktree path) for - /// via the injected resolver, never throwing (a lookup failure must not - /// fail the launch — it degrades to omitting the working directory, so the pod falls back to its - /// static AgentHost__WorkingDirectory env default). - /// - private async Task ResolveWorkingDirectoryAsync(string runId, CancellationToken ct) - { - if (_submittingUserResolver is null) - return null; - - try - { - return await _submittingUserResolver.GetWorkingDirectoryAsync(runId, ct).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger.LogWarning( - ex, - "KubernetesSandboxExecutor: failed to resolve working directory for run {RunId}; " + - "AgentHost__WorkingDirectory env default will be used.", - runId); - return null; - } - } - /// (AgentHostWarmPoolRef, replicas: 2). No spec.env is injected — the v0.5.0 - /// controller bypasses warm pool adoption whenever spec.env or - /// spec.volumeClaimTemplates are present. All static config lives in the SandboxTemplate - /// or agenthost-config ConfigMap. The per-run context (RunId / UserId / TurnBearerToken / - /// KV secret name) is delivered after bind via POST /configure - /// (). - /// - private async Task CreateAgentHostClaimAsync( - string claimName, string warmPoolName, string? workingDirectory, string runId, CancellationToken ct) - { - var annotations = new Dictionary - { - // Persist the ORIGINAL run id so the reaper can recover it from an orphaned claim (the - // claim name is a lossy 12-char derivation) and delete run-scoped side artifacts such as - // the per-run preview-runner credential (spec-006 decouple-preview). - [SandboxClaimConventions.RunIdAnnotation] = runId, - }; - if (!string.IsNullOrWhiteSpace(workingDirectory)) - annotations["agentweaver.io/working-directory"] = workingDirectory; - - var manifest = new - { - apiVersion = $"{ApiGroup}/{ApiVersion}", - kind = "SandboxClaim", - metadata = new - { - name = claimName, - @namespace = _options.Namespace, - annotations = annotations.Count == 0 ? null : annotations, - }, - spec = new - { - // v0.5.0 v1beta1 SandboxClaimSpec: spec.warmPoolRef.name references the - // SandboxWarmPool to bind from. sandboxTemplateRef+warmpool were the - // v0.4.x/v1alpha1 deprecated fields. - warmPoolRef = new { name = warmPoolName }, - lifecycle = new { ttlSecondsAfterFinished = _options.TimeoutSeconds, shutdownPolicy = "Delete" }, - }, - }; - - // Idempotent create with bounded transient-fault retry (issue #230). A mid-flight connection - // reset can commit the SandboxClaim server-side BEFORE we observe the response, so the retry - // may see a 409 for OUR OWN create — handled attempt-awarely below. - for (var attempt = 1; ; attempt++) - { - try - { - await _client.CustomObjects.CreateNamespacedCustomObjectAsync( - manifest, ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, - cancellationToken: ct).ConfigureAwait(false); - return true; - } - catch (HttpOperationException ex) when (ex.Response?.StatusCode == System.Net.HttpStatusCode.Conflict) - { - if (attempt > 1) - { - // Retry-409: a transient reset committed our create server-side before we saw the - // response, and this retry now observes our own claim. We own it → return true so - // the caller registers the turn token and runs /configure exactly as on a 200, - // rather than taking the silent "reuse pre-existing claim" path (which would leave - // the pod un-configured and token-less). - _logger.LogInformation( - "KubernetesSandboxExecutor: SandboxClaim {Claim} returned 409 on retry attempt {Attempt}; " + - "treating as our own create that committed before a transient reset — configuring it.", - claimName, attempt); - return true; - } - - // First-attempt 409: a genuinely pre-existing claim owned by an earlier launch. - _logger.LogInformation( - "KubernetesSandboxExecutor: SandboxClaim {Claim} already exists; waiting for existing claim", - claimName); - return false; - } - catch (Exception ex) when (attempt < MaxK8sAttempts && IsTransientK8sFault(ex, ct)) - { - var delay = BackoffWithJitter(attempt); - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: transient fault creating SandboxClaim {Claim} on attempt " + - "{Attempt}/{Max}; retrying in {DelayMs}ms.", - claimName, attempt, MaxK8sAttempts, (int)delay.TotalMilliseconds); - await Task.Delay(delay, ct).ConfigureAwait(false); - } - } - } - - // ── Transient Kubernetes API resilience (issue #230) ────────────────────────── - - /// - /// Executes an idempotent Kubernetes API call with a bounded retry ( - /// total attempts) over transient faults only — a mid-flight connection reset - /// (SocketException 104 → IOException → HttpRequestException), a 429/5xx from the API server, or an - /// HttpClient timeout. Caller cancellation is never retried and aborts the backoff immediately - /// (await Task.Delay(delay, ct)). Non-transient faults (e.g. 404/409/422) propagate on the - /// first attempt. MUST NOT wrap non-idempotent calls (e.g. the AgentHost POST /configure, - /// whose second delivery 409-hard-fails). - /// - private async Task ExecuteK8sWithRetryAsync( - Func> operation, CancellationToken ct) - { - for (var attempt = 1; ; attempt++) - { - try - { - return await operation(ct).ConfigureAwait(false); - } - catch (Exception ex) when (attempt < MaxK8sAttempts && IsTransientK8sFault(ex, ct)) - { - var delay = BackoffWithJitter(attempt); - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: transient Kubernetes API fault on attempt {Attempt}/{Max}; " + - "retrying in {DelayMs}ms.", attempt, MaxK8sAttempts, (int)delay.TotalMilliseconds); - await Task.Delay(delay, ct).ConfigureAwait(false); - } - } - } - - /// - /// Exponential backoff (~250ms · 2^(attempt-1), capped at ~2s) plus 0-250ms jitter to de-sync - /// concurrent launches retrying against the same API server after a blip. - /// - private static TimeSpan BackoffWithJitter(int attempt) - { - var baseMs = Math.Min(250 * (1 << (attempt - 1)), 2000); - var jitterMs = Random.Shared.Next(0, 250); - return TimeSpan.FromMilliseconds(baseMs + jitterMs); - } - - /// - /// True only for faults worth retrying an idempotent k8s call over: 429/5xx from the API server, - /// a socket/IO connection reset (directly or nested in an inner exception), or an HttpClient - /// timeout ( with no caller cancellation). Caller - /// cancellation short-circuits to false so a genuine cancel is never retried. A 409 Conflict is - /// intentionally NOT transient here — it is handled separately (idempotent create semantics). - /// - private static bool IsTransientK8sFault(Exception ex, CancellationToken ct) - { - if (ct.IsCancellationRequested) return false; // caller cancel — never retry - switch (ex) - { - case HttpOperationException k when k.Response is not null: - var s = (int)k.Response.StatusCode; - return s == 429 || s >= 500; // 409 handled separately, NOT here - case HttpRequestException: return true; - case IOException: return true; - case OperationCanceledException: // includes TaskCanceledException (HttpClient timeout) - return !ct.IsCancellationRequested; - } - for (Exception? i = ex.InnerException; i is not null; i = i.InnerException) - if (i is SocketException or IOException) return true; - return false; - } - - private async Task TryGetAgentHostClaimWorkingDirectoryAsync(string claimName, CancellationToken ct) - { - try - { - var raw = await _client.CustomObjects.GetNamespacedCustomObjectAsync( - ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, - cancellationToken: ct).ConfigureAwait(false); - var json = JsonSerializer.Serialize(raw); - using var doc = JsonDocument.Parse(json); - if (doc.RootElement.TryGetProperty("metadata", out var meta) && - meta.TryGetProperty("annotations", out var ann) && - ann.TryGetProperty("agentweaver.io/working-directory", out var wd) && - wd.ValueKind == JsonValueKind.String) - return wd.GetString(); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to read working-directory annotation for claim {Claim}", - claimName); - } - - return null; - } - - /// - /// Resolves the run owner's GitHub access token from the API-side token store so it can be - /// forwarded in the /configure body. The kata VM pod cannot reach Azure AD or Key Vault - /// (Cilium FQDN policies use eBPF interception that doesn't cross the guest kernel boundary). - /// Never throws — a lookup failure degrades gracefully: the pod will attempt the KV fetch itself - /// (which may fail) rather than causing a hard launch failure here. - /// - private async Task ResolveGitHubAccessTokenAsync( - GitHubTokenScope scope, - string userId, - CancellationToken ct) - { - // Prefer the refresh-aware provider (issue #523): a fresh AgentHost pod launched late in a - // long-running assembly (e.g. the Build & Test gate, well after the run's earlier subtask - // stages) can be handed a near-expiry or already-expired access token if we only ever read - // the raw stored entry — the pod's "fast path" trusts a pre-resolved token unconditionally - // and never re-validates it against Key Vault or GitHub. Routing through - // GetValidAccessTokenAsync mirrors GitHubCopilotClientFactory.CreateClientAsync and - // transparently rotates the token before it is handed to the pod. - if (_accessTokenProvider is not null) - { - try - { - var refreshed = await _accessTokenProvider.GetValidAccessTokenAsync(scope, ct) - .ConfigureAwait(false); - if (string.IsNullOrEmpty(refreshed)) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: refresh-aware GitHub token provider returned no valid credential " + - "for {UserId} (scope {Scope}); refusing raw-token fallback.", - userId, - scope.Key); - } - return refreshed; - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to resolve/refresh GitHub token for {UserId} via " + - "IGitHubAccessTokenProvider (scope {Scope}); refusing raw-token fallback.", - userId, - scope.Key); - return null; - } - } - - if (_tokenStore is null) - return null; - - try - { - var entry = await _tokenStore.GetAsync(scope, ct).ConfigureAwait(false); - if (entry.Status == GitHubTokenStatus.SignedIn && !string.IsNullOrEmpty(entry.AccessToken)) - return entry.AccessToken; - } - catch (Exception ex) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to pre-resolve GitHub token for {UserId} — pod will fall back to KV.", - userId); - } - - return null; - } - - /// - /// Injects the per-run context into an already-warm AgentHost pod via its one-time - /// POST /configure endpoint. The pod then fetches ONLY - /// from Key Vault (its configured user's token) and runs SetupAsync. The endpoint is guarded by - /// NetworkPolicy (ingress to AgentHost pods restricted to API/worker), not the TurnBearerToken - /// (which is itself delivered here). Idempotency: a second call returns 409 and is treated as a - /// hard launch failure. - /// - private async Task CallAgentHostConfigureAsync( - string podIp, int port, string runId, string userId, string turnBearerToken, - string kvUserSecretName, GitHubTokenScope tokenScope, string? gitHubAccessToken, - string? repositoryAccessToken, - string? sharedWorkingDirectory, - AgentHostLaunchContext launchContext, - string? projectId, - string? agentName, - CancellationToken ct) - { - if (_httpClientFactory is null) - { - // No HttpClient available (unit tests). Mirrors the readiness-probe null-skip; in-cluster - // the factory is always present, so this never short-circuits a real launch. - _logger.LogWarning( - "KubernetesSandboxExecutor: no IHttpClientFactory — skipping /configure for run {RunId}.", - runId); - return null; - } - - var scheme = AgentHostEndpoint.Scheme(_options.RequireMtls); - var configureUrl = $"{scheme}://{podIp}:{port}/configure"; - - // Mint a FRESH per-run preview-runner credential (spec-006 decouple-preview, BLOCKER A). - // Delivered in-memory via this /configure body ONLY (never pod env/file), and persisted to the - // run secret store so any replica can re-fetch it for reconcile/keepalive. Durably deleted on - // pod release. Every launch/relaunch mints a new value — the old one is never reused. - var previewRunnerCredential = await MintPreviewRunnerCredentialAsync(runId, ct).ConfigureAwait(false); - - var body = new - { - runId, - userId, - turnBearerToken, - kvUserSecretName, - gitHubAccessToken, - repositoryAccessToken, - callerBearerToken = launchContext.CallerBearerToken, - // Keep the legacy property during rolling upgrades; new AgentHosts prefer the explicit - // sharedWorkingDirectory descriptor and create any local workspace inside the pod. - workingDirectory = sharedWorkingDirectory, - sharedWorkingDirectory, - previewRunnerCredential, - purpose = launchContext.Purpose.ToString(), - launchContext.SourceRepositoryPath, - launchContext.SourceRef, - launchContext.BaseCommitSha, - launchContext.ExpectedTreeHash, - workspaceMode = launchContext.WorkspaceMode.ToString(), - launchContext.ScratchRoot, - launchContext.CommitAuthorName, - launchContext.CommitAuthorEmail, - // Per-run AutoApproveTools flag (bug #221). Resolved from the API-side run-options store - // keyed by the child runId; defaults false when the store is unavailable (unit tests). - autoApproveTools = _runOptions?.Get(runId).AutoApproveTools ?? false, - // Per-run project/agent identity (#335). Delivered so the in-pod agent's tool schema - // includes the Agentweaver API tools (record_memory, get_memory, submit_decision, - // list_decisions, list_inbox). Warm pods boot with an empty static AgentHost__ProjectId - // /AgentName, so without these the memory/decision tools never reach the agent. - projectId, - agentName, - }; - - _logger.LogInformation( - "KubernetesSandboxExecutor: configuring AgentHost pod for run {RunId} at {Url}", - runId, configureUrl); - - using var client = _httpClientFactory.CreateClient(HttpAgentHostReadinessProbe.HttpClientName); - using var response = await client - .PostAsJsonAsync(configureUrl, body, ct) - .ConfigureAwait(false); - var detail = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); - - if (!response.IsSuccessStatusCode) - { - var reason = "agenthost_configure_failed"; - try - { - using var document = JsonDocument.Parse(detail); - if (document.RootElement.TryGetProperty("error", out var error) - && error.ValueKind == JsonValueKind.String - && !string.IsNullOrWhiteSpace(error.GetString())) - reason = error.GetString()!; - } - catch (JsonException) - { - // Plain-text legacy errors keep the generic typed reason. - } - - if (string.Equals( - reason, - "agenthost_configure_copilot_unauthorized", - StringComparison.Ordinal) && - _accessTokenProvider is not null) - { - var refreshed = await _accessTokenProvider - .RefreshAfterUnauthorizedAsync(tokenScope, gitHubAccessToken, ct) - .ConfigureAwait(false); - if (!string.IsNullOrWhiteSpace(refreshed) && - !string.Equals(refreshed, gitHubAccessToken, StringComparison.Ordinal)) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: AgentHost /configure rejected the Copilot credential for run {RunId}; " + - "scope {Scope} was refreshed and the pod must be recreated (recoveryAttempt=1, maxRecoveryAttempts=1).", - runId, - tokenScope.Key); - throw new AgentHostConfigureException( - "agenthost_configure_copilot_token_refreshed", - $"AgentHost /configure rejected the Copilot credential for run '{runId}'. " + - "The credential was refreshed; recreate the one-time-configured pod and retry once.", - (int)response.StatusCode, - retryable: true, - recoveryAction: "recreate_pod_with_refreshed_credential"); - } - - _logger.LogWarning( - "KubernetesSandboxExecutor: AgentHost /configure rejected the Copilot credential for run {RunId}; " + - "scope {Scope} could not produce a different refreshed credential, so the failure is not retryable.", - runId, - tokenScope.Key); - } - - throw new AgentHostConfigureException( - reason, - $"AgentHost /configure for run '{runId}' failed: HTTP {(int)response.StatusCode} {detail}", - (int)response.StatusCode); - } - - if (string.IsNullOrWhiteSpace(detail)) - return null; - - try - { - using var document = JsonDocument.Parse(detail); - if (document.RootElement.TryGetProperty("effectiveWorkingDirectory", out var path) - && path.ValueKind == JsonValueKind.String - && !string.IsNullOrWhiteSpace(path.GetString())) - { - return path.GetString(); - } - } - catch (JsonException ex) - { - _logger.LogWarning( - ex, - "KubernetesSandboxExecutor: AgentHost /configure for run {RunId} returned an invalid success body; preview will use the shared working directory.", - runId); - } - - return null; - } - - /// - /// Mints and persists a fresh per-run preview-runner credential and returns it for in-memory - /// delivery via /configure. Returns when no secret store is - /// available (unit tests) — the pod then relies on the turn token only. The persisted key is - /// derived deterministically from the run id () - /// so the release-time delete matches (spec-006 decouple-preview, BLOCKER A). - /// - private async Task MintPreviewRunnerCredentialAsync(string runId, CancellationToken ct) - { - if (_secretStore is null) - return string.Empty; - - var credential = Preview.PreviewRunnerCredential.Mint(); - var key = Preview.PreviewRunnerCredential.SecretKey(runId); - try - { - await _secretStore.SetSecretAsync(key, credential, etag: null, ct).ConfigureAwait(false); - _logger.LogInformation( - "KubernetesSandboxExecutor: minted per-run preview-runner credential for run {RunId}", runId); - return credential; - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - // Best-effort: a persist failure must not fail the launch. The pod still receives the - // credential in-memory (same-process affinity uses the turn token anyway), but a - // cross-replica reconcile could not re-fetch it — acceptable degradation. - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to persist preview-runner credential for run {RunId}; " + - "delivering in-memory only.", runId); - return credential; - } - } - - /// - /// Durably deletes the per-run preview-runner credential from the run secret store. No-op when - /// absent ( ignores a missing key). Never throws — - /// a delete failure must not break terminal cleanup. Called on EVERY terminal path (happy - /// release + crash/timeout/failed-run via the pod-release seam) so the credential's durable - /// lifetime is bounded by the pod's (spec-006 decouple-preview, RESIDUAL rev3 gap). - /// - private async Task DeletePreviewRunnerCredentialAsync(string runId, CancellationToken ct) - { - if (_secretStore is null) - return; - - try - { - await _secretStore.DeleteSecretAsync(Preview.PreviewRunnerCredential.SecretKey(runId), ct) - .ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to delete preview-runner credential for run {RunId} (best-effort)", - runId); - } - } - - private async Task RevokeRepositoryCredentialAsync(string runId, CancellationToken ct) - { - if (_repositoryCredentials is null) - return; - - try - { - await _repositoryCredentials.RevokeAsync(runId, ct).ConfigureAwait(false); - } - catch (Exception ex) when (ex is not OperationCanceledException) - { - _logger.LogWarning( - ex, - "KubernetesSandboxExecutor: failed to revoke repository credential for run {RunId}", - runId); - } - } - - /// - /// Waits for the AgentHost SandboxClaim to bind while emitting periodic - /// heartbeats into the CHILD run's event - /// stream. Scheduling is Kubernetes' job: a claim may sit unbound (pod Pending) for a while until - /// a node frees up or the pool autoscales — that is FINE and must not fail the run (issue #217). - /// The heartbeat keeps the parent coordinator's subtask-stall timer alive during that legitimate - /// wait, mirroring the #212 tool.approval_pending heartbeat. Best-effort: if no - /// is wired (unit tests) this degrades to a plain - /// . - /// - private async Task WaitForBoundWithProvisioningHeartbeatAsync( - string runId, string claimName, CancellationToken ct) - { - if (_runEventStream is null) - return await WaitForBoundAsync(claimName, ct).ConfigureAwait(false); - - var boundTask = WaitForBoundAsync(claimName, ct); - while (true) - { - var delayTask = Task.Delay(SandboxProvisioningHeartbeatInterval, ct); - var completed = await Task.WhenAny(boundTask, delayTask).ConfigureAwait(false); - if (ReferenceEquals(completed, boundTask)) - return await boundTask.ConfigureAwait(false); // propagates the bound pod name / any error - - // The claim is still unbound after the heartbeat interval — emit a non-terminal - // heartbeat so the coordinator's stall window resets while Kubernetes schedules the pod. - await delayTask.ConfigureAwait(false); // observe cancellation - await EmitProvisioningPendingAsync(runId, claimName, ct).ConfigureAwait(false); - } - } - - /// - /// Appends a single heartbeat to - /// 's durable event stream. Best-effort: a stream-append failure is - /// logged and swallowed so it can never fail a launch that Kubernetes would otherwise admit. - /// - private async Task EmitProvisioningPendingAsync(string runId, string claimName, CancellationToken ct) - { - try - { - await _runEventStream!.AppendAsync(runId, new RunEvent(0, EventTypes.SandboxProvisioningPending, new - { - claimName, - timestamp_utc = DateTimeOffset.UtcNow.ToString("O"), - }), ct).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: failed to emit sandbox.provisioning_pending heartbeat for run {RunId} (best-effort)", - runId); - } - } - - /// - /// Parses a Kubernetes CPU quantity into whole cores. Handles plain cores ("24", - /// "1.5") and the millicore suffix ("500m" = 0.5 cores). Returns - /// for an unrecognized format. - /// - internal static bool TryParseCpu(string? value, out double cores) - { - cores = 0; - if (string.IsNullOrWhiteSpace(value)) - return false; - - value = value.Trim(); - if (value.EndsWith("m", StringComparison.Ordinal)) - { - var millis = value[..^1]; - if (double.TryParse(millis, System.Globalization.NumberStyles.Float, - System.Globalization.CultureInfo.InvariantCulture, out var m)) - { - cores = m / 1000.0; - return true; - } - return false; - } - - return double.TryParse(value, System.Globalization.NumberStyles.Float, - System.Globalization.CultureInfo.InvariantCulture, out cores); - } - - /// - /// Reads the pod IP from the Kubernetes API after the claim is Bound. - /// Polls every 2 s until status.podIP is non-empty (pod has been scheduled - /// and assigned a network address). - /// - private async Task GetPodIpAsync(string podName, CancellationToken ct) - { - while (true) - { - ct.ThrowIfCancellationRequested(); - - var pod = await ExecuteK8sWithRetryAsync( - token => _client.CoreV1.ReadNamespacedPodAsync( - podName, _options.Namespace, cancellationToken: token), - ct).ConfigureAwait(false); - - var ip = pod?.Status?.PodIP; - if (!string.IsNullOrWhiteSpace(ip)) - return ip; - - _logger.LogDebug( - "KubernetesSandboxExecutor: waiting for pod IP of {Pod} (current: {Ip})", - podName, ip ?? "(none)"); - - await Task.Delay(2000, ct).ConfigureAwait(false); - } - } - - // ── Claim management ────────────────────────────────────────────────────────── - - private async Task CreateClaimAsync(string claimName, CancellationToken ct) - { - // The cluster service CIDR must be present in SandboxEgressCidrExclusions so - // sandbox NetworkPolicy does not accidentally allow in-cluster service egress. - var manifest = new - { - apiVersion = $"{ApiGroup}/{ApiVersion}", - kind = "SandboxClaim", - metadata = new { name = claimName, @namespace = _options.Namespace }, - spec = new - { - // v0.5.0 v1beta1 SandboxClaimSpec: spec.warmPoolRef.name references the - // SandboxWarmPool to bind from. sandboxTemplateRef+warmpool were the - // v0.4.x/v1alpha1 deprecated fields. - warmPoolRef = new { name = _options.WarmPoolRef }, - lifecycle = new { ttlSecondsAfterFinished = _options.TimeoutSeconds, shutdownPolicy = "Delete" }, - }, - }; - - try - { - await _client.CustomObjects.CreateNamespacedCustomObjectAsync( - manifest, ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, - cancellationToken: ct).ConfigureAwait(false); - return true; - } - catch (HttpOperationException ex) when (ex.Response?.StatusCode == System.Net.HttpStatusCode.Conflict) - { - _logger.LogInformation( - "KubernetesSandboxExecutor: SandboxClaim {Claim} already exists; waiting for existing claim", - claimName); - return false; - } - } - - /// - /// Polls every 2 s until the claim's Ready condition is True; returns the bound - /// pod name from status.sandbox.name. - /// - private async Task WaitForBoundAsync(string claimName, CancellationToken ct) - { - while (true) - { - ct.ThrowIfCancellationRequested(); - - var raw = await ExecuteK8sWithRetryAsync( - token => _client.CustomObjects.GetNamespacedCustomObjectAsync( - ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, - cancellationToken: token), - ct).ConfigureAwait(false); - - var json = JsonSerializer.Serialize(raw); - using var doc = JsonDocument.Parse(json); - - // Surface a controller reconcile failure (e.g. "exceeded quota") as a deterministic - // launch failure with a precise reason instead of polling until the caller times out. - var reconcilerError = SandboxClaimConventions.TryGetReconcilerError(doc.RootElement); - if (reconcilerError is not null) - { - _logger.LogWarning( - "KubernetesSandboxExecutor: claim {Claim} reconcile failed: {Error}", - claimName, reconcilerError); - throw new AgentHostPodReconcilerErrorException( - $"SandboxClaim '{claimName}' could not be provisioned: {reconcilerError}"); - } - - var podName = SandboxClaimConventions.TryGetBoundPodName(doc.RootElement); - if (!string.IsNullOrEmpty(podName)) - return podName; - - await Task.Delay(2000, ct); - } - } - - private async Task DeleteClaimAsync(string claimName, CancellationToken ct = default) - { - try - { - await _client.CustomObjects.DeleteNamespacedCustomObjectAsync( - ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, cancellationToken: ct); - _logger.LogInformation( - "KubernetesSandboxExecutor: deleted claim {Claim}", claimName); - } - catch (Exception ex) - { - _logger.LogWarning(ex, - "KubernetesSandboxExecutor: could not delete claim {Claim} (best-effort)", claimName); - } - } - - // ── Command execution ───────────────────────────────────────────────────────── - - private async Task ExecInPodAsync( - string podName, SandboxCommand command, string podWorkingDirectory, CancellationToken ct) - { - const int maxOutputBytes = 4 * 1024 * 1024; - - var shellScript = BuildShellScript(command, podWorkingDirectory); - - var ws = await _client.WebSocketNamespacedPodExecAsync( - podName, _options.Namespace, - new[] { "/bin/sh", "-c", shellScript }, - container: ContainerName, - stdin: false, stdout: true, stderr: true, tty: false, - cancellationToken: ct); - - using var demux = new StreamDemuxer(ws, StreamType.RemoteCommand); - demux.Start(); - - using var stdoutStream = demux.GetStream(ChannelIndex.StdOut, null); - using var stderrStream = demux.GetStream(ChannelIndex.StdErr, null); - // Channel 3 (Error) carries the terminal v1.Status payload with the real exit code. - using var statusStream = demux.GetStream(ChannelIndex.Error, null); - - var stdoutTask = ReadBoundedAsync(stdoutStream, maxOutputBytes, ct); - var stderrTask = ReadBoundedAsync(stderrStream, maxOutputBytes, ct); - var statusTask = ReadBoundedAsync(statusStream, maxOutputBytes, ct); - - await Task.WhenAll(stdoutTask, stderrTask, statusTask); - - var (stdoutBytes, stdoutTruncated) = await stdoutTask; - var (stderrBytes, stderrTruncated) = await stderrTask; - var (statusBytes, _) = await statusTask; - - var stdout = SandboxOutputRedactor.Default.Redact(Encoding.UTF8.GetString(stdoutBytes)); - var stderr = SandboxOutputRedactor.Default.Redact(Encoding.UTF8.GetString(stderrBytes)); - var exitCode = ParseExitCode(Encoding.UTF8.GetString(statusBytes)); - - return new SandboxExecResult( - exitCode, stdout, stderr, false, stdoutTruncated || stderrTruncated); - } - - /// - /// Reads up to from a stream, stopping at the cap. - /// Returns the bytes collected and whether the output was truncated. - /// - private static async Task<(byte[] Bytes, bool Truncated)> ReadBoundedAsync( - Stream stream, int maxBytes, CancellationToken ct) - { - using var buffer = new MemoryStream(); - var chunk = new byte[8192]; - bool truncated = false; - int read; - while ((read = await stream.ReadAsync(chunk, ct)) > 0) - { - int remaining = maxBytes - (int)buffer.Length; - if (remaining <= 0) { truncated = true; break; } - int take = Math.Min(read, remaining); - buffer.Write(chunk, 0, take); - if (take < read) { truncated = true; break; } - } - return (buffer.ToArray(), truncated); - } - - /// - /// Parses the terminal v1.Status JSON emitted on channel 3. - /// status: "Success" → exit 0. status: "Failure" → the ExitCode - /// cause from details.causes (defaulting to 1 if not present). - /// - private static int ParseExitCode(string statusJson) - { - if (string.IsNullOrWhiteSpace(statusJson)) - return 0; - - try - { - using var doc = JsonDocument.Parse(statusJson); - var root = doc.RootElement; - - var status = root.TryGetProperty("status", out var s) ? s.GetString() : null; - if (string.Equals(status, "Success", StringComparison.OrdinalIgnoreCase)) - return 0; - - if (root.TryGetProperty("details", out var details) && - details.TryGetProperty("causes", out var causes) && - causes.ValueKind == JsonValueKind.Array) - { - foreach (var cause in causes.EnumerateArray()) - { - var reason = cause.TryGetProperty("reason", out var r) ? r.GetString() : null; - if (string.Equals(reason, "ExitCode", StringComparison.OrdinalIgnoreCase) && - cause.TryGetProperty("message", out var m) && - int.TryParse(m.GetString(), out var code)) - return code; - } - } - - // Failure status with no parseable ExitCode cause → non-zero. - return 1; - } - catch (JsonException) - { - return 0; - } - } - - private string ResolvePodWorkingDirectory(string requestedWorkingDirectory) - { - var mountPath = NormalizeUnixPath(_options.WorkspaceMountPath, forceAbsolute: true); - if (string.IsNullOrWhiteSpace(requestedWorkingDirectory)) - return mountPath; - - var requested = NormalizeUnixPath(requestedWorkingDirectory, forceAbsolute: false); - if (IsSameOrChildPath(requested, mountPath)) - return requested; - - throw new InvalidOperationException( - $"Kubernetes sandbox working directory '{requestedWorkingDirectory}' is not under mounted workspace '{mountPath}'. " + - "Configure Workspace:PersistentVolume:MountRoot/Workspace:Path to match the workspace PVC mount used by sandbox pods."); - } - - private static bool IsSameOrChildPath(string path, string root) => - string.Equals(path, root, StringComparison.Ordinal) - || (root == "/" && path.StartsWith("/", StringComparison.Ordinal)) - || path.StartsWith(root + "/", StringComparison.Ordinal); - - private static string NormalizeUnixPath(string path, bool forceAbsolute) - { - var normalized = path.Trim().Replace('\\', '/'); - while (normalized.Contains("//", StringComparison.Ordinal)) - normalized = normalized.Replace("//", "/", StringComparison.Ordinal); - if (forceAbsolute && !normalized.StartsWith("/", StringComparison.Ordinal)) - normalized = "/" + normalized; - return normalized.Length > 1 ? normalized.TrimEnd('/') : normalized; - } - - private static string BuildShellScript(SandboxCommand command, string podWorkingDirectory) - { - var sb = new StringBuilder(); - - if (command.Environment is { Count: > 0 }) - { - foreach (var (key, value) in command.Environment) - sb.AppendLine($"export {key}={ShellSingleQuote(value)}"); - } - - sb.AppendLine($"cd {ShellSingleQuote(podWorkingDirectory)}"); - - sb.Append(command.CommandLine); - return sb.ToString(); - } - - private static string ShellSingleQuote(string s) => - "'" + s.Replace("'", "'\\''") + "'"; -} +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; +using System.Net.Http.Json; +using System.Net.Sockets; +using System.Text.Json; +using Agentweaver.Api.Auth; +using Agentweaver.Api.Infrastructure; +using Agentweaver.AgentRuntime.Workflow; +using Agentweaver.Domain; +using k8s; +using k8s.Autorest; +using Agentweaver.SandboxExec; +using Microsoft.Extensions.Logging; + +namespace Agentweaver.Api.Sandbox; + +/// +/// Configures the Kubernetes SandboxClaim backend. +/// Bound from the Sandbox:Kubernetes configuration section. +/// +public sealed class KubernetesSandboxOptions +{ + public string Namespace { get; init; } = "agentweaver"; + public string TemplateRef { get; init; } = "agentweaver-sandbox"; + /// + /// SandboxWarmPool the generic command-exec claim binds to. In the v1beta1 CRD a + /// SandboxClaim references a SandboxWarmPool (spec.warmPoolRef.name), + /// which in turn references the SandboxTemplate. Default: agentweaver-sandbox. + /// + public string WarmPoolRef { get; init; } = "agentweaver-sandbox"; + /// Path where the shared workspace PVC is mounted inside API and sandbox pods. + public string WorkspaceMountPath { get; init; } = "/workspace"; + /// SandboxClaim TTL. Command timeouts are capped below this so controller GC cannot interrupt exec. + public int TimeoutSeconds { get; init; } = 600; + /// Cluster service CIDR that must be excluded by sandbox egress policy. + public string? ServiceCidr { get; init; } + public IReadOnlyList SandboxEgressCidrExclusions { get; init; } = []; + + // ── Pod-per-run AgentHost lifecycle options (spec §9 / Q3 hybrid) ───────── + + /// + /// SandboxWarmPool the AgentHost (pod-per-run) claim binds to in the v0.5.0 v1beta1 CRD + /// (spec.warmPoolRef.name). The pool itself references the AgentHost SandboxTemplate. + /// Default: agentweaver-agent-host. + /// + public string AgentHostWarmPoolRef { get; init; } = "agentweaver-agent-host"; + + /// + /// Port the AgentHost Kestrel listener binds to inside the pod. + /// Worker builds the A2A endpoint as http://<podIP>:<AgentHostPort><AgentHostA2APath>. + /// TLS/mTLS termination is owned by Link (H1) — leave hook here for cert wiring. + /// Default: 8088. + /// + public int AgentHostPort { get; init; } = 8088; + + /// + /// A2A path prefix mounted by MapA2AHttpJson inside the AgentHost pod. + /// Must match AgentHost:A2APath set in the pod's configuration. + /// Default: /a2a/agent. + /// + public string AgentHostA2APath { get; init; } = "/a2a/agent"; + + /// + /// When (default) the AgentHost A2A endpoint uses https with + /// mTLS (H1). When (PoC only) it uses plain http. Drives the + /// scheme via and is injected into the pod as + /// AgentHost__RequireMtls. Config key: Sandbox:AgentHost:RequireMtls. + /// + public bool RequireMtls { get; init; } = true; + + // ── AgentHost readiness gate (A2A cold-start race) ─────────────────────── + + /// + /// Path the AgentHost exposes for liveness/readiness on . The executor + /// polls {scheme}://{podIP}:{port}{AgentHostHealthzPath} after the claim binds and BEFORE + /// returning the A2A endpoint, so the worker never sends the first turn into the Kestrel boot + /// window (which would be refused). Default: /healthz. + /// + public string AgentHostHealthzPath { get; init; } = "/healthz"; + + /// + /// Maximum time to wait for the AgentHost to start serving + /// before failing the launch deterministically. Default: 90s (covers cold-start Kestrel bind). + /// + public int AgentHostReadyTimeoutSeconds { get; init; } = 90; + + /// Interval between AgentHost readiness probe attempts. Default: 1000ms. + public int AgentHostReadyPollIntervalMs { get; init; } = 1000; + + /// + /// Minimum age before the orphan reaper may delete an AgentHost claim that is absent from the + /// active-run map. Config key: Sandbox:Kubernetes:AgentHostClaimCreationGraceSeconds. + /// The effective value is floored above . + /// Default: 300s. + /// + public int AgentHostClaimCreationGraceSeconds { get; init; } = 300; + + /// + /// Azure Key Vault URI injected into AgentHost pods as AgentHost__KeyVaultUri so the + /// warm pod can fetch the run owner's GitHub token via workload identity at /configure-time + /// (Option C). Sourced from the API's own KV config (Auth:TokenStore:KeyVaultUri). When + /// null/empty the env var is omitted and the pod falls back to the CSI file-mount path. + /// + public string? KvUri { get; init; } +} + +/// +/// Top-level sandbox runtime options bound from the Sandbox configuration section +/// (not under Sandbox:Kubernetes). Controls the agent-execution mode and +/// the pod-release-on-suspend behaviour (Q3 hybrid). +/// +public sealed class SandboxRuntimeOptions +{ + /// + /// Agent execution mode. + /// + /// in-api (default) — run agents in-process; instant rollback path (§4.7.6). + /// pod-per-run — launch a per-run AgentHost sandbox pod; activate A2A transport. + /// + /// + public string AgentExecutionMode { get; init; } = "in-api"; + + /// + /// When true (default) and is pod-per-run, + /// the AgentHost pod is released (SandboxClaim deleted) whenever the MAF graph suspends + /// at a RequestPort (HITL/review gate) or the coordinator idles awaiting children. + /// Set to false to keep the pod warm across suspension (lower resume latency, higher + /// resource cost; recommended only for short-wait HITL in dev/staging). + /// + public bool ReleasePodOnSuspend { get; init; } = true; + + /// + public bool IsPodPerRun => + string.Equals(AgentExecutionMode, "pod-per-run", StringComparison.OrdinalIgnoreCase); +} + +/// +/// Executes sandboxed commands inside a pre-warmed Kubernetes pod obtained via a +/// SandboxClaim CRD. Lifecycle: +/// +/// Create a SandboxClaim resource (adopts a warm pod from the pool). +/// Poll until the claim transitions to phase: Bound and reports a pod name. +/// Run the command via pod-exec (Kubernetes WebSocket exec API). +/// Delete the claim on completion (controller GC cleans up the pod and service). +/// +/// Automatically selected by the API when KUBERNETES_SERVICE_HOST is present +/// (see ). +/// +internal sealed class KubernetesSandboxExecutor : ISandboxExecutor, IAgentHostPodLifecycle +{ + private const string ApiGroup = SandboxClaimConventions.ApiGroup; + private const string ApiVersion = SandboxClaimConventions.ApiVersion; + private const string ClaimPlural = SandboxClaimConventions.ClaimPlural; + private const string ContainerName = "agentweaver-sandbox"; + + /// + /// Bounded attempt count for — the total number of + /// tries (initial + retries) for a transient Kubernetes API fault (issue #230). A transient + /// connection reset (SocketException 104 → IOException → HttpRequestException) that used to fail + /// a subtask fatally is now retried with exponential backoff + jitter. + /// + private const int MaxK8sAttempts = 3; + + /// + /// Cadence for the heartbeat emitted while an + /// AgentHost SandboxClaim is still being provisioned (unbound). Must stay well under the + /// parent coordinator's Coordinator:SubtaskStallTimeoutMinutes (default 5 min) so each + /// provisioning wait window is punctuated by an event that keeps the outbound stream flowing and + /// resets the stall timer (issue #217, mirrors the #212 tool.approval_pending heartbeat cadence). + /// + internal static readonly TimeSpan SandboxProvisioningHeartbeatInterval = TimeSpan.FromSeconds(20); + + private readonly IKubernetes _client; + private readonly KubernetesSandboxOptions _options; + private readonly ILogger _logger; + private readonly IPodNameRegistry? _podRegistry; + private readonly IAgentHostTurnTokenRegistry? _turnTokenRegistry; + private readonly Security.IRunAuthorshipCapabilityStore? _authorshipCapabilityStore; + // Polls the AgentHost /healthz after bind and before returning the endpoint, closing the + // A2A cold-start race (pod Running ~20-30s before Kestrel binds :8088). Null in unit tests + // that only assert the claim body → readiness gate is skipped. + private readonly IAgentHostReadinessProbe? _readinessProbe; + // Resolves the run's submitting user so the pod can be scoped (via /configure) to the run owner's + // Copilot-entitled token instead of the installation token. Null when the run→user lookup is + // unavailable. + private readonly IRunSubmittingUserResolver? _submittingUserResolver; + // Used to POST /configure to the warm pod after bind (warm-pool deferred-config path). Null in + // unit tests → the /configure call is skipped (same null-skip convention as the readiness probe). + private readonly IHttpClientFactory? _httpClientFactory; + // Resolves the run owner's GitHub token so the API can pass it in /configure, avoiding the need + // for the kata VM pod to call Azure AD or Key Vault (blocked by Cilium FQDN policies). + private readonly IGitHubTokenStore? _tokenStore; + private readonly IGitHubTokenScopeProvider? _tokenScopeProvider; + // Refresh-aware token accessor (issue #523): a Build & Test gate can launch its AgentHost pod for + // the FIRST time (a fresh pod, not yet /configure'd for this run) many minutes after the run's + // earlier subtask stages — long enough for the submitting user's Copilot-entitled OAuth access + // token to cross its expiry skew window. Reading the raw entry via IGitHubTokenStore.GetAsync (as + // ResolveGitHubAccessTokenAsync previously did) can hand a stale/expired access token to the pod, + // which the pod then trusts unconditionally (its "fast path" skips its own Key Vault fetch + // whenever a pre-resolved token arrives) — producing GitHubCopilotUnauthorizedException at + // /configure. Routing through the same GetValidAccessTokenAsync used by GitHubCopilotClientFactory + // ensures a near-expiry token is transparently rotated before being handed to a newly-launched pod. + // Null in unit tests → falls back to the raw (non-refreshing) token store read. When present, + // it is authoritative: a null/failed refresh must never fall back to the rejected raw token. + private readonly IGitHubAccessTokenProvider? _accessTokenProvider; + // Replica-safe run secret store used to persist the per-run preview-runner credential so a + // reconcile/keepalive on either API replica can re-fetch it, and to durably DELETE it on pod + // release (spec-006 decouple-preview, BLOCKER A / RESIDUAL). Null in unit tests → no minting. + private readonly ISecretStore? _secretStore; + // Durable run-event log used to emit sandbox.provisioning_pending heartbeats into the CHILD run's + // stream while its AgentHost claim is still being scheduled by Kubernetes (unbound). Keeps the + // parent coordinator's stall timer alive during a legitimately-long Pending wait (issue #217). + // Null in unit tests → the heartbeat is skipped (same null-skip convention as the readiness probe). + private readonly IRunEventStream? _runEventStream; + // Source of the per-run AutoApproveTools flag propagated to the warm pod via /configure (bug + // #221). Null in unit tests → the flag defaults false (same null-skip convention as above). + private readonly IRunOptionsStore? _runOptions; + private readonly RunRepositoryCredentialRegistry? _repositoryCredentials; + // First-class preview lifecycle reconciler. ReleaseAgentHostPodAsync derives durable + // Previewable/PreviewActive state and applies all retention or cleanup effects before deciding + // whether to delete the claim. + private readonly Agentweaver.Api.Sandbox.Preview.ISandboxPreviewService? _previewService; + + public bool IsRealIsolation => true; + public string BackendName => "kubernetes-sandbox-claim"; + public string SelectionReason => + "Kubernetes-native sandbox via SandboxClaim warm pool (Kata VM isolation, NetworkPolicy egress restriction)."; + public bool HasNetworkWarning => false; + public string? NetworkWarningMessage => null; + + internal KubernetesSandboxExecutor( + IKubernetes client, + KubernetesSandboxOptions options, + ILogger logger, + IPodNameRegistry? podRegistry = null, + IAgentHostTurnTokenRegistry? turnTokenRegistry = null, + IAgentHostReadinessProbe? readinessProbe = null, + IRunSubmittingUserResolver? submittingUserResolver = null, + IHttpClientFactory? httpClientFactory = null, + IGitHubTokenStore? tokenStore = null, + ISecretStore? secretStore = null, + IRunEventStream? runEventStream = null, + IRunOptionsStore? runOptions = null, + RunRepositoryCredentialRegistry? repositoryCredentials = null, + IGitHubAccessTokenProvider? accessTokenProvider = null, + Agentweaver.Api.Sandbox.Preview.ISandboxPreviewService? previewService = null, + IGitHubTokenScopeProvider? tokenScopeProvider = null, + Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null) + { + _client = client; + _options = options; + _logger = logger; + _podRegistry = podRegistry; + _turnTokenRegistry = turnTokenRegistry; + _readinessProbe = readinessProbe; + _submittingUserResolver = submittingUserResolver; + _httpClientFactory = httpClientFactory; + _tokenStore = tokenStore; + _tokenScopeProvider = tokenScopeProvider; + _secretStore = secretStore; + _runEventStream = runEventStream; + _runOptions = runOptions; + _repositoryCredentials = repositoryCredentials; + _accessTokenProvider = accessTokenProvider; + _previewService = previewService; + _authorshipCapabilityStore = authorshipCapabilityStore; + } + + public async Task ExecuteAsync( + SandboxCommand command, CancellationToken ct = default) + { + // Use the Agentweaver run ID as the claim name when available so the pod can be + // looked up by run ID later (preview port-forward). Fall back to a random ID. + var claimName = string.IsNullOrEmpty(command.AgentweaverRunId) + ? $"run-{Guid.NewGuid():N}"[..20] + : SandboxClaimConventions.DeriveRunCommandClaimName(command.AgentweaverRunId); + + var requestedTimeoutMs = command.TimeoutMs > 0 + ? command.TimeoutMs + : _options.TimeoutSeconds * 1000; + var maxCommandTimeoutMs = Math.Max(1000, (_options.TimeoutSeconds * 1000) - 30_000); + var timeoutMs = Math.Min(requestedTimeoutMs, maxCommandTimeoutMs); + if (timeoutMs < requestedTimeoutMs) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: command timeout clamped from {RequestedMs}ms to {TimeoutMs}ms so it stays below SandboxClaim TTL ({TtlSeconds}s)", + requestedTimeoutMs, timeoutMs, _options.TimeoutSeconds); + } + + string podWorkingDirectory; + try + { + podWorkingDirectory = ResolvePodWorkingDirectory(command.WorkingDirectory); + } + catch (Exception ex) + { + _logger.LogError(ex, + "KubernetesSandboxExecutor: invalid workspace path {WorkingDirectory}; configured mount is {WorkspaceMountPath}", + command.WorkingDirectory, _options.WorkspaceMountPath); + return new SandboxExecResult(1, "", ex.Message, false, false); + } + + _logger.LogInformation( + "KubernetesSandboxExecutor: using workspace path {WorkspacePath} for claim {Claim} (requested {RequestedWorkingDirectory})", + podWorkingDirectory, claimName, command.WorkingDirectory); + + using var linked = CancellationTokenSource.CreateLinkedTokenSource(ct); + linked.CancelAfter(timeoutMs); + var token = linked.Token; + var claimCreated = false; + + try + { + _logger.LogInformation( + "KubernetesSandboxExecutor: creating SandboxClaim {Claim}", claimName); + claimCreated = await CreateClaimAsync(claimName, token); + + var podName = await WaitForBoundAsync(claimName, token); + _logger.LogInformation( + "KubernetesSandboxExecutor: claim {Claim} bound to pod {Pod}", claimName, podName); + + // Register pod name so PortForwardService can locate it by Agentweaver run ID. + // Run-scoped mappings are cleared by run lifecycle cleanup, not per command, so + // preview tunnels can remain available for the whole run while the claim TTL is valid. + if (!string.IsNullOrEmpty(command.AgentweaverRunId)) + _podRegistry?.Register(command.AgentweaverRunId, podName); + + return await ExecInPodAsync(podName, command, podWorkingDirectory, token); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: timed out waiting for claim {Claim}", claimName); + return new SandboxExecResult(-1, "", "Timed out waiting for sandbox pod.", true, false); + } + finally + { + if (claimCreated && string.IsNullOrEmpty(command.AgentweaverRunId)) + await DeleteClaimAsync(claimName); + else if (claimCreated) + _logger.LogDebug( + "KubernetesSandboxExecutor: retaining SandboxClaim {Claim} for run {RunId} preview until run cleanup or TTL", + claimName, command.AgentweaverRunId); + } + } + + public async IAsyncEnumerable StreamAsync( + SandboxCommand command, + [EnumeratorCancellation] CancellationToken ct = default) + { + var result = await ExecuteAsync(command, ct); + foreach (var line in result.Stdout.Split('\n')) + yield return new SandboxOutputChunk(SandboxOutputStream.Stdout, line); + if (!string.IsNullOrEmpty(result.Stderr)) + foreach (var line in result.Stderr.Split('\n')) + yield return new SandboxOutputChunk(SandboxOutputStream.Stderr, line); + yield return new SandboxOutputChunk(SandboxOutputStream.ExitCode, result.ExitCode.ToString()); + } + + // ── IAgentHostPodLifecycle — pod-per-run lifecycle (spec §9 / Q3) ───────────── + + /// + public Task LaunchAgentHostPodAsync(string runId, CancellationToken ct = default) => + LaunchAgentHostPodAsync(runId, new AgentHostLaunchContext(SharedWorkingDirectory: null), ct); + + /// + public Task LaunchAgentHostPodAsync( + string runId, + string? workingDirectoryOverride, + CancellationToken ct = default) => + LaunchAgentHostPodAsync( + runId, + new AgentHostLaunchContext(SharedWorkingDirectory: workingDirectoryOverride), + ct); + + /// + public async Task LaunchAgentHostPodAsync( + string runId, + AgentHostLaunchContext launchContext, + CancellationToken ct = default) + { + var claimName = SandboxClaimConventions.DeriveAgentHostClaimName(runId); + var requestedWorkingDirectory = string.IsNullOrWhiteSpace(launchContext.SharedWorkingDirectory) + ? null + : Path.GetFullPath(launchContext.SharedWorkingDirectory); + + _logger.LogInformation( + "KubernetesSandboxExecutor: launching AgentHost pod for run {RunId} via claim {Claim}", + runId, claimName); + + // Resolve the run's submitting user so the pod can scope GitHub Copilot auth to that user's + // signed-in token. The user's Key Vault secret name (Option C warm-pool path) is derived here + // and delivered to the pod via /configure — never another user's secret. + var submittingUser = await ResolveSubmittingUserAsync(runId, ct).ConfigureAwait(false); + if (string.IsNullOrWhiteSpace(submittingUser)) + { + throw new InvalidOperationException( + $"Cannot launch AgentHost pod for run '{runId}' without a submitting user; " + + "the /configure call must scope the pod to the run owner's Key Vault token."); + } + + _logger.LogInformation( + "KubernetesSandboxExecutor: resolved submitting user for run {RunId}; will configure pod via /configure.", + runId); + + var (configProjectId, configAgentName) = _submittingUserResolver is not null + ? await _submittingUserResolver.GetRunIdentityAsync(runId, ct).ConfigureAwait(false) + : (null, null); + + // ghtok-user--{base32(userId)} — the SAME mapping the API uses when persisting the token to KV. + // With Entra sign-in the user's credentials live under the ACTIVE linked GitHub identity's + // scope (user-link:{oid}:{login}), so resolve the effective scope rather than assuming the + // legacy per-user scope, which is never written in that mode. + var effectiveScope = _tokenScopeProvider is not null + ? await _tokenScopeProvider + .ResolveAsync(submittingUser!, configProjectId, ct) + .ConfigureAwait(false) + : _tokenStore is IEffectiveGitHubTokenScopeResolver scopeResolver + ? await scopeResolver.ResolveEffectiveScopeAsync(submittingUser!, ct).ConfigureAwait(false) + : GitHubTokenScope.ForUser(submittingUser!); + var kvUserSecretName = KeyVaultSecretStore.SanitizeKey(effectiveScope.Key); + var turnToken = Convert.ToBase64String(RandomNumberGenerator.GetBytes(32)); + var claimCreated = false; + try + { + // Bind to the SHARED, pre-warmed AgentHost warm pool (replicas: 2). No per-run SPC, + // SandboxTemplate, or warm pool — the pod is already warm and gets its per-run context + // via the /configure POST below. + claimCreated = await CreateAgentHostClaimAsync( + claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); + + if (!claimCreated && launchContext.Purpose == AgentHostPurpose.OperatorAssistant) + { + // Every operator turn carries the CURRENT browser/platform bearer. An orphaned + // claim from a crashed prior turn is already configured with the old credential + // and /configure is intentionally one-shot, so it must never be reused. + _logger.LogInformation( + "KubernetesSandboxExecutor: recreating existing AgentHost claim {Claim} for a fresh operator-assistant caller credential.", + claimName); + await DeleteClaimAsync(claimName).ConfigureAwait(false); + _podRegistry?.Unregister(runId); + _turnTokenRegistry?.UnregisterTurnToken(runId); + await Task.Delay(1000, ct).ConfigureAwait(false); + claimCreated = await CreateAgentHostClaimAsync( + claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); + if (!claimCreated) + { + throw new InvalidOperationException( + $"AgentHost claim '{claimName}' was deleted to refresh the operator-assistant caller credential, " + + "but the replacement create still conflicted."); + } + } + else if (!claimCreated && launchContext.WorkspaceMode != ExecutionWorkspaceMode.Shared) + { + _logger.LogInformation( + "KubernetesSandboxExecutor: recreating existing AgentHost claim {Claim} for immutable pod-local workspace configuration (mode={Mode}).", + claimName, + launchContext.WorkspaceMode); + await DeleteClaimAsync(claimName).ConfigureAwait(false); + _podRegistry?.Unregister(runId); + _turnTokenRegistry?.UnregisterTurnToken(runId); + await Task.Delay(1000, ct).ConfigureAwait(false); + claimCreated = await CreateAgentHostClaimAsync( + claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); + if (!claimCreated) + { + throw new InvalidOperationException( + $"AgentHost claim '{claimName}' was deleted for immutable pod-local workspace configuration, " + + "but the replacement create still conflicted."); + } + } + else if (!claimCreated && requestedWorkingDirectory is not null) + { + var existingWorkingDirectory = await TryGetAgentHostClaimWorkingDirectoryAsync(claimName, ct) + .ConfigureAwait(false); + var sameWorktree = string.Equals( + existingWorkingDirectory, requestedWorkingDirectory, StringComparison.Ordinal); + var hasTurnToken = !string.IsNullOrWhiteSpace(_turnTokenRegistry?.TryGetTurnToken(runId)); + + if (!sameWorktree || !hasTurnToken) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: existing AgentHost claim {Claim} for run {RunId} " + + "is not reusable (sameWorktree={SameWorktree}, hasTurnToken={HasTurnToken}); recreating.", + claimName, runId, sameWorktree, hasTurnToken); + await DeleteClaimAsync(claimName).ConfigureAwait(false); + _podRegistry?.Unregister(runId); + _turnTokenRegistry?.UnregisterTurnToken(runId); + await Task.Delay(1000, ct).ConfigureAwait(false); + claimCreated = await CreateAgentHostClaimAsync( + claimName, _options.AgentHostWarmPoolRef, requestedWorkingDirectory, runId, ct).ConfigureAwait(false); + if (!claimCreated) + { + throw new InvalidOperationException( + $"AgentHost claim '{claimName}' for run '{runId}' was deleted for worktree reconfiguration, " + + "but the replacement create still conflicted. Retrying later avoids reusing a token-less or stale pod."); + } + } + } + + var podName = await WaitForBoundWithProvisioningHeartbeatAsync(runId, claimName, ct).ConfigureAwait(false); + _logger.LogInformation( + "KubernetesSandboxExecutor: AgentHost claim {Claim} bound to pod {Pod}", claimName, podName); + + // Register also persists sandbox.execution_pod.bound into the shared RunEvents store so + // graph snapshots/deltas on any API replica can resolve the execution pod. + _podRegistry?.Register(runId, podName); + if (claimCreated) + _turnTokenRegistry?.RegisterTurnToken(runId, turnToken); + + var activeTurnToken = claimCreated + ? turnToken + : _turnTokenRegistry?.TryGetTurnToken(runId); + if (_authorshipCapabilityStore is not null && !string.IsNullOrWhiteSpace(activeTurnToken)) + { + await _authorshipCapabilityStore.RegisterAsync( + runId, activeTurnToken, DateTimeOffset.UtcNow.AddDays(1), ct).ConfigureAwait(false); + } + + var podIp = await GetPodIpAsync(podName, ct).ConfigureAwait(false); + + var endpointUrl = AgentHostEndpoint.Build( + _options.RequireMtls, podIp, _options.AgentHostPort, _options.AgentHostA2APath); + + // A2A cold-start gate: the claim binds when the pod is Running, but the AgentHost Kestrel + // listener takes ~20-30s more to bind :8088. Without this wait the worker's first A2A POST + // hits a closed port → "Connection refused" → the run fails mid-turn. Poll /healthz until the + // app is actually serving so a not-yet-ready pod is a deterministic LAUNCH failure instead. + // NOTE: a warm/standby pod serves /healthz BEFORE /configure (the readiness gate exempts + // /configure), so this confirms reachability prior to injecting the run context. + if (_readinessProbe is not null) + { + var scheme = AgentHostEndpoint.Scheme(_options.RequireMtls); + var readinessUrl = + $"{scheme}://{podIp}:{_options.AgentHostPort}{_options.AgentHostHealthzPath}"; + + _logger.LogInformation( + "KubernetesSandboxExecutor: waiting for AgentHost readiness for run {RunId} at {Url}", + runId, readinessUrl); + + try + { + await _readinessProbe.WaitUntilReadyAsync(readinessUrl, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + throw new InvalidOperationException( + $"AgentHost pod '{podName}' for run '{runId}' did not become ready at {readinessUrl} " + + $"within {_options.AgentHostReadyTimeoutSeconds}s; failing the launch.", ex); + } + } + + // Warm-pool deferred /configure: inject the per-run RunId/UserId/TurnBearerToken and the + // KV secret name into the already-warm pod, which then runs SetupAsync and becomes ready. + // Normal roles use the shared orchestration worktree. Local workspace modes carry + // immutable source refs; AgentHost creates their effective root inside execution-scratch. + if (claimCreated) + { + var repositoryAccessToken = _repositoryCredentials is null + ? null + : await _repositoryCredentials.MintAsync(runId, ct).ConfigureAwait(false); + var effectiveWorkingDirectory = await CallAgentHostConfigureAsync( + podIp, _options.AgentHostPort, runId, submittingUser, turnToken, kvUserSecretName, + effectiveScope, + await ResolveGitHubAccessTokenAsync(effectiveScope, submittingUser, ct).ConfigureAwait(false), + repositoryAccessToken, + requestedWorkingDirectory ?? await ResolveWorkingDirectoryAsync(runId, ct).ConfigureAwait(false), + launchContext, + configProjectId, + configAgentName, + ct) + .ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(effectiveWorkingDirectory)) + _podRegistry?.RegisterEffectiveWorkingDirectory(runId, effectiveWorkingDirectory); + } + else + { + _logger.LogInformation( + "KubernetesSandboxExecutor: reusing already-configured AgentHost claim {Claim} for run {RunId}", + claimName, runId); + } + + _podRegistry?.RegisterAgentEndpoint(runId, endpointUrl); + + _logger.LogInformation( + "KubernetesSandboxExecutor: AgentHost A2A endpoint for run {RunId} = {Endpoint}", + runId, endpointUrl); + + return endpointUrl; + } + catch + { + if (claimCreated) + await DeleteClaimAsync(claimName).ConfigureAwait(false); + _podRegistry?.Unregister(runId); + _turnTokenRegistry?.UnregisterTurnToken(runId); + if (_authorshipCapabilityStore is not null) + { + await _authorshipCapabilityStore.RemoveAsync(runId, CancellationToken.None) + .ConfigureAwait(false); + } + // Crash/timeout during launch: delete any credential minted before the failure so it is + // never left behind (spec-006 decouple-preview, RESIDUAL rev3 gap). + await DeletePreviewRunnerCredentialAsync(runId, CancellationToken.None).ConfigureAwait(false); + await RevokeRepositoryCredentialAsync(runId, CancellationToken.None).ConfigureAwait(false); + throw; + } + } + + /// + public async Task ReleaseAgentHostPodAsync(string runId, CancellationToken ct = default) + { + var claimName = SandboxClaimConventions.DeriveAgentHostClaimName(runId); + + // Issue #542: if a live preview is still active for this run, releasing the pod here (at the + // originating subtask's turn end) would 404 the preview URL before any human-review gate or + // demo viewer can open it. Defer the claim delete while the preview is alive; the preview's own + // idle/max expiry + the reaper will eventually reap the pod, so this cannot leak. + if (_previewService is not null && + await _previewService.ReconcilePreviewLifecycleAsync(runId, ct).ConfigureAwait(false) + == Agentweaver.Api.Sandbox.Preview.PreviewLifecycleState.PreviewActive) + { + _logger.LogInformation( + "KubernetesSandboxExecutor: deferring AgentHost pod release for run {RunId} (claim " + + "{Claim}) — a live preview is still active; the preview idle/max expiry will reap it.", + runId, claimName); + return; + } + + _logger.LogInformation( + "KubernetesSandboxExecutor: releasing AgentHost pod for run {RunId} (claim {Claim})", + runId, claimName); + + await DeleteClaimAsync(claimName, ct).ConfigureAwait(false); + _podRegistry?.Unregister(runId); + _turnTokenRegistry?.UnregisterTurnToken(runId); + if (_authorshipCapabilityStore is not null) + await _authorshipCapabilityStore.RemoveAsync(runId, ct).ConfigureAwait(false); + await DeletePreviewRunnerCredentialAsync(runId, ct).ConfigureAwait(false); + await RevokeRepositoryCredentialAsync(runId, ct).ConfigureAwait(false); + + _logger.LogInformation( + "KubernetesSandboxExecutor: AgentHost pod released for run {RunId}", runId); + } + + /// + /// Resolves the submitting user for via the injected resolver, never + /// throwing (a lookup failure must not fail the launch — it degrades to omitting the user id). + /// + private async Task ResolveSubmittingUserAsync(string runId, CancellationToken ct) + { + if (_submittingUserResolver is null) + return null; + + try + { + return await _submittingUserResolver.GetSubmittingUserAsync(runId, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "KubernetesSandboxExecutor: failed to resolve submitting user for run {RunId}; " + + "AgentHost__UserId will be omitted.", + runId); + return null; + } + } + + /// + /// Resolves the per-run working directory (shared orchestration worktree path) for + /// via the injected resolver, never throwing (a lookup failure must not + /// fail the launch — it degrades to omitting the working directory, so the pod falls back to its + /// static AgentHost__WorkingDirectory env default). + /// + private async Task ResolveWorkingDirectoryAsync(string runId, CancellationToken ct) + { + if (_submittingUserResolver is null) + return null; + + try + { + return await _submittingUserResolver.GetWorkingDirectoryAsync(runId, ct).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "KubernetesSandboxExecutor: failed to resolve working directory for run {RunId}; " + + "AgentHost__WorkingDirectory env default will be used.", + runId); + return null; + } + } + /// (AgentHostWarmPoolRef, replicas: 2). No spec.env is injected — the v0.5.0 + /// controller bypasses warm pool adoption whenever spec.env or + /// spec.volumeClaimTemplates are present. All static config lives in the SandboxTemplate + /// or agenthost-config ConfigMap. The per-run context (RunId / UserId / TurnBearerToken / + /// KV secret name) is delivered after bind via POST /configure + /// (). + /// + private async Task CreateAgentHostClaimAsync( + string claimName, string warmPoolName, string? workingDirectory, string runId, CancellationToken ct) + { + var annotations = new Dictionary + { + // Persist the ORIGINAL run id so the reaper can recover it from an orphaned claim (the + // claim name is a lossy 12-char derivation) and delete run-scoped side artifacts such as + // the per-run preview-runner credential (spec-006 decouple-preview). + [SandboxClaimConventions.RunIdAnnotation] = runId, + }; + if (!string.IsNullOrWhiteSpace(workingDirectory)) + annotations["agentweaver.io/working-directory"] = workingDirectory; + + var manifest = new + { + apiVersion = $"{ApiGroup}/{ApiVersion}", + kind = "SandboxClaim", + metadata = new + { + name = claimName, + @namespace = _options.Namespace, + annotations = annotations.Count == 0 ? null : annotations, + }, + spec = new + { + // v0.5.0 v1beta1 SandboxClaimSpec: spec.warmPoolRef.name references the + // SandboxWarmPool to bind from. sandboxTemplateRef+warmpool were the + // v0.4.x/v1alpha1 deprecated fields. + warmPoolRef = new { name = warmPoolName }, + lifecycle = new { ttlSecondsAfterFinished = _options.TimeoutSeconds, shutdownPolicy = "Delete" }, + }, + }; + + // Idempotent create with bounded transient-fault retry (issue #230). A mid-flight connection + // reset can commit the SandboxClaim server-side BEFORE we observe the response, so the retry + // may see a 409 for OUR OWN create — handled attempt-awarely below. + for (var attempt = 1; ; attempt++) + { + try + { + await _client.CustomObjects.CreateNamespacedCustomObjectAsync( + manifest, ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, + cancellationToken: ct).ConfigureAwait(false); + return true; + } + catch (HttpOperationException ex) when (ex.Response?.StatusCode == System.Net.HttpStatusCode.Conflict) + { + if (attempt > 1) + { + // Retry-409: a transient reset committed our create server-side before we saw the + // response, and this retry now observes our own claim. We own it → return true so + // the caller registers the turn token and runs /configure exactly as on a 200, + // rather than taking the silent "reuse pre-existing claim" path (which would leave + // the pod un-configured and token-less). + _logger.LogInformation( + "KubernetesSandboxExecutor: SandboxClaim {Claim} returned 409 on retry attempt {Attempt}; " + + "treating as our own create that committed before a transient reset — configuring it.", + claimName, attempt); + return true; + } + + // First-attempt 409: a genuinely pre-existing claim owned by an earlier launch. + _logger.LogInformation( + "KubernetesSandboxExecutor: SandboxClaim {Claim} already exists; waiting for existing claim", + claimName); + return false; + } + catch (Exception ex) when (attempt < MaxK8sAttempts && IsTransientK8sFault(ex, ct)) + { + var delay = BackoffWithJitter(attempt); + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: transient fault creating SandboxClaim {Claim} on attempt " + + "{Attempt}/{Max}; retrying in {DelayMs}ms.", + claimName, attempt, MaxK8sAttempts, (int)delay.TotalMilliseconds); + await Task.Delay(delay, ct).ConfigureAwait(false); + } + } + } + + // ── Transient Kubernetes API resilience (issue #230) ────────────────────────── + + /// + /// Executes an idempotent Kubernetes API call with a bounded retry ( + /// total attempts) over transient faults only — a mid-flight connection reset + /// (SocketException 104 → IOException → HttpRequestException), a 429/5xx from the API server, or an + /// HttpClient timeout. Caller cancellation is never retried and aborts the backoff immediately + /// (await Task.Delay(delay, ct)). Non-transient faults (e.g. 404/409/422) propagate on the + /// first attempt. MUST NOT wrap non-idempotent calls (e.g. the AgentHost POST /configure, + /// whose second delivery 409-hard-fails). + /// + private async Task ExecuteK8sWithRetryAsync( + Func> operation, CancellationToken ct) + { + for (var attempt = 1; ; attempt++) + { + try + { + return await operation(ct).ConfigureAwait(false); + } + catch (Exception ex) when (attempt < MaxK8sAttempts && IsTransientK8sFault(ex, ct)) + { + var delay = BackoffWithJitter(attempt); + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: transient Kubernetes API fault on attempt {Attempt}/{Max}; " + + "retrying in {DelayMs}ms.", attempt, MaxK8sAttempts, (int)delay.TotalMilliseconds); + await Task.Delay(delay, ct).ConfigureAwait(false); + } + } + } + + /// + /// Exponential backoff (~250ms · 2^(attempt-1), capped at ~2s) plus 0-250ms jitter to de-sync + /// concurrent launches retrying against the same API server after a blip. + /// + private static TimeSpan BackoffWithJitter(int attempt) + { + var baseMs = Math.Min(250 * (1 << (attempt - 1)), 2000); + var jitterMs = Random.Shared.Next(0, 250); + return TimeSpan.FromMilliseconds(baseMs + jitterMs); + } + + /// + /// True only for faults worth retrying an idempotent k8s call over: 429/5xx from the API server, + /// a socket/IO connection reset (directly or nested in an inner exception), or an HttpClient + /// timeout ( with no caller cancellation). Caller + /// cancellation short-circuits to false so a genuine cancel is never retried. A 409 Conflict is + /// intentionally NOT transient here — it is handled separately (idempotent create semantics). + /// + private static bool IsTransientK8sFault(Exception ex, CancellationToken ct) + { + if (ct.IsCancellationRequested) return false; // caller cancel — never retry + switch (ex) + { + case HttpOperationException k when k.Response is not null: + var s = (int)k.Response.StatusCode; + return s == 429 || s >= 500; // 409 handled separately, NOT here + case HttpRequestException: return true; + case IOException: return true; + case OperationCanceledException: // includes TaskCanceledException (HttpClient timeout) + return !ct.IsCancellationRequested; + } + for (Exception? i = ex.InnerException; i is not null; i = i.InnerException) + if (i is SocketException or IOException) return true; + return false; + } + + private async Task TryGetAgentHostClaimWorkingDirectoryAsync(string claimName, CancellationToken ct) + { + try + { + var raw = await _client.CustomObjects.GetNamespacedCustomObjectAsync( + ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, + cancellationToken: ct).ConfigureAwait(false); + var json = JsonSerializer.Serialize(raw); + using var doc = JsonDocument.Parse(json); + if (doc.RootElement.TryGetProperty("metadata", out var meta) && + meta.TryGetProperty("annotations", out var ann) && + ann.TryGetProperty("agentweaver.io/working-directory", out var wd) && + wd.ValueKind == JsonValueKind.String) + return wd.GetString(); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to read working-directory annotation for claim {Claim}", + claimName); + } + + return null; + } + + /// + /// Resolves the run owner's GitHub access token from the API-side token store so it can be + /// forwarded in the /configure body. The kata VM pod cannot reach Azure AD or Key Vault + /// (Cilium FQDN policies use eBPF interception that doesn't cross the guest kernel boundary). + /// Never throws — a lookup failure degrades gracefully: the pod will attempt the KV fetch itself + /// (which may fail) rather than causing a hard launch failure here. + /// + private async Task ResolveGitHubAccessTokenAsync( + GitHubTokenScope scope, + string userId, + CancellationToken ct) + { + // Prefer the refresh-aware provider (issue #523): a fresh AgentHost pod launched late in a + // long-running assembly (e.g. the Build & Test gate, well after the run's earlier subtask + // stages) can be handed a near-expiry or already-expired access token if we only ever read + // the raw stored entry — the pod's "fast path" trusts a pre-resolved token unconditionally + // and never re-validates it against Key Vault or GitHub. Routing through + // GetValidAccessTokenAsync mirrors GitHubCopilotClientFactory.CreateClientAsync and + // transparently rotates the token before it is handed to the pod. + if (_accessTokenProvider is not null) + { + try + { + var refreshed = await _accessTokenProvider.GetValidAccessTokenAsync(scope, ct) + .ConfigureAwait(false); + if (string.IsNullOrEmpty(refreshed)) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: refresh-aware GitHub token provider returned no valid credential " + + "for {UserId} (scope {Scope}); refusing raw-token fallback.", + userId, + scope.Key); + } + return refreshed; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to resolve/refresh GitHub token for {UserId} via " + + "IGitHubAccessTokenProvider (scope {Scope}); refusing raw-token fallback.", + userId, + scope.Key); + return null; + } + } + + if (_tokenStore is null) + return null; + + try + { + var entry = await _tokenStore.GetAsync(scope, ct).ConfigureAwait(false); + if (entry.Status == GitHubTokenStatus.SignedIn && !string.IsNullOrEmpty(entry.AccessToken)) + return entry.AccessToken; + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to pre-resolve GitHub token for {UserId} — pod will fall back to KV.", + userId); + } + + return null; + } + + /// + /// Injects the per-run context into an already-warm AgentHost pod via its one-time + /// POST /configure endpoint. The pod then fetches ONLY + /// from Key Vault (its configured user's token) and runs SetupAsync. The endpoint is guarded by + /// NetworkPolicy (ingress to AgentHost pods restricted to API/worker), not the TurnBearerToken + /// (which is itself delivered here). Idempotency: a second call returns 409 and is treated as a + /// hard launch failure. + /// + private async Task CallAgentHostConfigureAsync( + string podIp, int port, string runId, string userId, string turnBearerToken, + string kvUserSecretName, GitHubTokenScope tokenScope, string? gitHubAccessToken, + string? repositoryAccessToken, + string? sharedWorkingDirectory, + AgentHostLaunchContext launchContext, + string? projectId, + string? agentName, + CancellationToken ct) + { + if (_httpClientFactory is null) + { + // No HttpClient available (unit tests). Mirrors the readiness-probe null-skip; in-cluster + // the factory is always present, so this never short-circuits a real launch. + _logger.LogWarning( + "KubernetesSandboxExecutor: no IHttpClientFactory — skipping /configure for run {RunId}.", + runId); + return null; + } + + var scheme = AgentHostEndpoint.Scheme(_options.RequireMtls); + var configureUrl = $"{scheme}://{podIp}:{port}/configure"; + + // Mint a FRESH per-run preview-runner credential (spec-006 decouple-preview, BLOCKER A). + // Delivered in-memory via this /configure body ONLY (never pod env/file), and persisted to the + // run secret store so any replica can re-fetch it for reconcile/keepalive. Durably deleted on + // pod release. Every launch/relaunch mints a new value — the old one is never reused. + var previewRunnerCredential = await MintPreviewRunnerCredentialAsync(runId, ct).ConfigureAwait(false); + + var body = new + { + runId, + userId, + turnBearerToken, + kvUserSecretName, + gitHubAccessToken, + repositoryAccessToken, + callerBearerToken = launchContext.CallerBearerToken, + // Keep the legacy property during rolling upgrades; new AgentHosts prefer the explicit + // sharedWorkingDirectory descriptor and create any local workspace inside the pod. + workingDirectory = sharedWorkingDirectory, + sharedWorkingDirectory, + previewRunnerCredential, + purpose = launchContext.Purpose.ToString(), + launchContext.SourceRepositoryPath, + launchContext.SourceRef, + launchContext.BaseCommitSha, + launchContext.ExpectedTreeHash, + workspaceMode = launchContext.WorkspaceMode.ToString(), + launchContext.ScratchRoot, + launchContext.CommitAuthorName, + launchContext.CommitAuthorEmail, + // Per-run AutoApproveTools flag (bug #221). Resolved from the API-side run-options store + // keyed by the child runId; defaults false when the store is unavailable (unit tests). + autoApproveTools = _runOptions?.Get(runId).AutoApproveTools ?? false, + // Per-run project/agent identity (#335). Delivered so the in-pod agent's tool schema + // includes the Agentweaver API tools (record_memory, get_memory, submit_decision, + // list_decisions, list_inbox). Warm pods boot with an empty static AgentHost__ProjectId + // /AgentName, so without these the memory/decision tools never reach the agent. + projectId, + agentName, + }; + + _logger.LogInformation( + "KubernetesSandboxExecutor: configuring AgentHost pod for run {RunId} at {Url}", + runId, configureUrl); + + using var client = _httpClientFactory.CreateClient(HttpAgentHostReadinessProbe.HttpClientName); + using var response = await client + .PostAsJsonAsync(configureUrl, body, ct) + .ConfigureAwait(false); + var detail = await response.Content.ReadAsStringAsync(ct).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + var reason = "agenthost_configure_failed"; + try + { + using var document = JsonDocument.Parse(detail); + if (document.RootElement.TryGetProperty("error", out var error) + && error.ValueKind == JsonValueKind.String + && !string.IsNullOrWhiteSpace(error.GetString())) + reason = error.GetString()!; + } + catch (JsonException) + { + // Plain-text legacy errors keep the generic typed reason. + } + + if (string.Equals( + reason, + "agenthost_configure_copilot_unauthorized", + StringComparison.Ordinal) && + _accessTokenProvider is not null) + { + var refreshed = await _accessTokenProvider + .RefreshAfterUnauthorizedAsync(tokenScope, gitHubAccessToken, ct) + .ConfigureAwait(false); + if (!string.IsNullOrWhiteSpace(refreshed) && + !string.Equals(refreshed, gitHubAccessToken, StringComparison.Ordinal)) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: AgentHost /configure rejected the Copilot credential for run {RunId}; " + + "scope {Scope} was refreshed and the pod must be recreated (recoveryAttempt=1, maxRecoveryAttempts=1).", + runId, + tokenScope.Key); + throw new AgentHostConfigureException( + "agenthost_configure_copilot_token_refreshed", + $"AgentHost /configure rejected the Copilot credential for run '{runId}'. " + + "The credential was refreshed; recreate the one-time-configured pod and retry once.", + (int)response.StatusCode, + retryable: true, + recoveryAction: "recreate_pod_with_refreshed_credential"); + } + + _logger.LogWarning( + "KubernetesSandboxExecutor: AgentHost /configure rejected the Copilot credential for run {RunId}; " + + "scope {Scope} could not produce a different refreshed credential, so the failure is not retryable.", + runId, + tokenScope.Key); + } + + throw new AgentHostConfigureException( + reason, + $"AgentHost /configure for run '{runId}' failed: HTTP {(int)response.StatusCode} {detail}", + (int)response.StatusCode); + } + + if (string.IsNullOrWhiteSpace(detail)) + return null; + + try + { + using var document = JsonDocument.Parse(detail); + if (document.RootElement.TryGetProperty("effectiveWorkingDirectory", out var path) + && path.ValueKind == JsonValueKind.String + && !string.IsNullOrWhiteSpace(path.GetString())) + { + return path.GetString(); + } + } + catch (JsonException ex) + { + _logger.LogWarning( + ex, + "KubernetesSandboxExecutor: AgentHost /configure for run {RunId} returned an invalid success body; preview will use the shared working directory.", + runId); + } + + return null; + } + + /// + /// Mints and persists a fresh per-run preview-runner credential and returns it for in-memory + /// delivery via /configure. Returns when no secret store is + /// available (unit tests) — the pod then relies on the turn token only. The persisted key is + /// derived deterministically from the run id () + /// so the release-time delete matches (spec-006 decouple-preview, BLOCKER A). + /// + private async Task MintPreviewRunnerCredentialAsync(string runId, CancellationToken ct) + { + if (_secretStore is null) + return string.Empty; + + var credential = Preview.PreviewRunnerCredential.Mint(); + var key = Preview.PreviewRunnerCredential.SecretKey(runId); + try + { + await _secretStore.SetSecretAsync(key, credential, etag: null, ct).ConfigureAwait(false); + _logger.LogInformation( + "KubernetesSandboxExecutor: minted per-run preview-runner credential for run {RunId}", runId); + return credential; + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + // Best-effort: a persist failure must not fail the launch. The pod still receives the + // credential in-memory (same-process affinity uses the turn token anyway), but a + // cross-replica reconcile could not re-fetch it — acceptable degradation. + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to persist preview-runner credential for run {RunId}; " + + "delivering in-memory only.", runId); + return credential; + } + } + + /// + /// Durably deletes the per-run preview-runner credential from the run secret store. No-op when + /// absent ( ignores a missing key). Never throws — + /// a delete failure must not break terminal cleanup. Called on EVERY terminal path (happy + /// release + crash/timeout/failed-run via the pod-release seam) so the credential's durable + /// lifetime is bounded by the pod's (spec-006 decouple-preview, RESIDUAL rev3 gap). + /// + private async Task DeletePreviewRunnerCredentialAsync(string runId, CancellationToken ct) + { + if (_secretStore is null) + return; + + try + { + await _secretStore.DeleteSecretAsync(Preview.PreviewRunnerCredential.SecretKey(runId), ct) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to delete preview-runner credential for run {RunId} (best-effort)", + runId); + } + } + + private async Task RevokeRepositoryCredentialAsync(string runId, CancellationToken ct) + { + if (_repositoryCredentials is null) + return; + + try + { + await _repositoryCredentials.RevokeAsync(runId, ct).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning( + ex, + "KubernetesSandboxExecutor: failed to revoke repository credential for run {RunId}", + runId); + } + } + + /// + /// Waits for the AgentHost SandboxClaim to bind while emitting periodic + /// heartbeats into the CHILD run's event + /// stream. Scheduling is Kubernetes' job: a claim may sit unbound (pod Pending) for a while until + /// a node frees up or the pool autoscales — that is FINE and must not fail the run (issue #217). + /// The heartbeat keeps the parent coordinator's subtask-stall timer alive during that legitimate + /// wait, mirroring the #212 tool.approval_pending heartbeat. Best-effort: if no + /// is wired (unit tests) this degrades to a plain + /// . + /// + private async Task WaitForBoundWithProvisioningHeartbeatAsync( + string runId, string claimName, CancellationToken ct) + { + if (_runEventStream is null) + return await WaitForBoundAsync(claimName, ct).ConfigureAwait(false); + + var boundTask = WaitForBoundAsync(claimName, ct); + while (true) + { + var delayTask = Task.Delay(SandboxProvisioningHeartbeatInterval, ct); + var completed = await Task.WhenAny(boundTask, delayTask).ConfigureAwait(false); + if (ReferenceEquals(completed, boundTask)) + return await boundTask.ConfigureAwait(false); // propagates the bound pod name / any error + + // The claim is still unbound after the heartbeat interval — emit a non-terminal + // heartbeat so the coordinator's stall window resets while Kubernetes schedules the pod. + await delayTask.ConfigureAwait(false); // observe cancellation + await EmitProvisioningPendingAsync(runId, claimName, ct).ConfigureAwait(false); + } + } + + /// + /// Appends a single heartbeat to + /// 's durable event stream. Best-effort: a stream-append failure is + /// logged and swallowed so it can never fail a launch that Kubernetes would otherwise admit. + /// + private async Task EmitProvisioningPendingAsync(string runId, string claimName, CancellationToken ct) + { + try + { + await _runEventStream!.AppendAsync(runId, new RunEvent(0, EventTypes.SandboxProvisioningPending, new + { + claimName, + timestamp_utc = DateTimeOffset.UtcNow.ToString("O"), + }), ct).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: failed to emit sandbox.provisioning_pending heartbeat for run {RunId} (best-effort)", + runId); + } + } + + /// + /// Parses a Kubernetes CPU quantity into whole cores. Handles plain cores ("24", + /// "1.5") and the millicore suffix ("500m" = 0.5 cores). Returns + /// for an unrecognized format. + /// + internal static bool TryParseCpu(string? value, out double cores) + { + cores = 0; + if (string.IsNullOrWhiteSpace(value)) + return false; + + value = value.Trim(); + if (value.EndsWith("m", StringComparison.Ordinal)) + { + var millis = value[..^1]; + if (double.TryParse(millis, System.Globalization.NumberStyles.Float, + System.Globalization.CultureInfo.InvariantCulture, out var m)) + { + cores = m / 1000.0; + return true; + } + return false; + } + + return double.TryParse(value, System.Globalization.NumberStyles.Float, + System.Globalization.CultureInfo.InvariantCulture, out cores); + } + + /// + /// Reads the pod IP from the Kubernetes API after the claim is Bound. + /// Polls every 2 s until status.podIP is non-empty (pod has been scheduled + /// and assigned a network address). + /// + private async Task GetPodIpAsync(string podName, CancellationToken ct) + { + while (true) + { + ct.ThrowIfCancellationRequested(); + + var pod = await ExecuteK8sWithRetryAsync( + token => _client.CoreV1.ReadNamespacedPodAsync( + podName, _options.Namespace, cancellationToken: token), + ct).ConfigureAwait(false); + + var ip = pod?.Status?.PodIP; + if (!string.IsNullOrWhiteSpace(ip)) + return ip; + + _logger.LogDebug( + "KubernetesSandboxExecutor: waiting for pod IP of {Pod} (current: {Ip})", + podName, ip ?? "(none)"); + + await Task.Delay(2000, ct).ConfigureAwait(false); + } + } + + // ── Claim management ────────────────────────────────────────────────────────── + + private async Task CreateClaimAsync(string claimName, CancellationToken ct) + { + // The cluster service CIDR must be present in SandboxEgressCidrExclusions so + // sandbox NetworkPolicy does not accidentally allow in-cluster service egress. + var manifest = new + { + apiVersion = $"{ApiGroup}/{ApiVersion}", + kind = "SandboxClaim", + metadata = new { name = claimName, @namespace = _options.Namespace }, + spec = new + { + // v0.5.0 v1beta1 SandboxClaimSpec: spec.warmPoolRef.name references the + // SandboxWarmPool to bind from. sandboxTemplateRef+warmpool were the + // v0.4.x/v1alpha1 deprecated fields. + warmPoolRef = new { name = _options.WarmPoolRef }, + lifecycle = new { ttlSecondsAfterFinished = _options.TimeoutSeconds, shutdownPolicy = "Delete" }, + }, + }; + + try + { + await _client.CustomObjects.CreateNamespacedCustomObjectAsync( + manifest, ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, + cancellationToken: ct).ConfigureAwait(false); + return true; + } + catch (HttpOperationException ex) when (ex.Response?.StatusCode == System.Net.HttpStatusCode.Conflict) + { + _logger.LogInformation( + "KubernetesSandboxExecutor: SandboxClaim {Claim} already exists; waiting for existing claim", + claimName); + return false; + } + } + + /// + /// Polls every 2 s until the claim's Ready condition is True; returns the bound + /// pod name from status.sandbox.name. + /// + private async Task WaitForBoundAsync(string claimName, CancellationToken ct) + { + while (true) + { + ct.ThrowIfCancellationRequested(); + + var raw = await ExecuteK8sWithRetryAsync( + token => _client.CustomObjects.GetNamespacedCustomObjectAsync( + ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, + cancellationToken: token), + ct).ConfigureAwait(false); + + var json = JsonSerializer.Serialize(raw); + using var doc = JsonDocument.Parse(json); + + // Surface a controller reconcile failure (e.g. "exceeded quota") as a deterministic + // launch failure with a precise reason instead of polling until the caller times out. + var reconcilerError = SandboxClaimConventions.TryGetReconcilerError(doc.RootElement); + if (reconcilerError is not null) + { + _logger.LogWarning( + "KubernetesSandboxExecutor: claim {Claim} reconcile failed: {Error}", + claimName, reconcilerError); + throw new AgentHostPodReconcilerErrorException( + $"SandboxClaim '{claimName}' could not be provisioned: {reconcilerError}"); + } + + var podName = SandboxClaimConventions.TryGetBoundPodName(doc.RootElement); + if (!string.IsNullOrEmpty(podName)) + return podName; + + await Task.Delay(2000, ct); + } + } + + private async Task DeleteClaimAsync(string claimName, CancellationToken ct = default) + { + try + { + await _client.CustomObjects.DeleteNamespacedCustomObjectAsync( + ApiGroup, ApiVersion, _options.Namespace, ClaimPlural, claimName, cancellationToken: ct); + _logger.LogInformation( + "KubernetesSandboxExecutor: deleted claim {Claim}", claimName); + } + catch (Exception ex) + { + _logger.LogWarning(ex, + "KubernetesSandboxExecutor: could not delete claim {Claim} (best-effort)", claimName); + } + } + + // ── Command execution ───────────────────────────────────────────────────────── + + private async Task ExecInPodAsync( + string podName, SandboxCommand command, string podWorkingDirectory, CancellationToken ct) + { + const int maxOutputBytes = 4 * 1024 * 1024; + + var shellScript = BuildShellScript(command, podWorkingDirectory); + + var ws = await _client.WebSocketNamespacedPodExecAsync( + podName, _options.Namespace, + new[] { "/bin/sh", "-c", shellScript }, + container: ContainerName, + stdin: false, stdout: true, stderr: true, tty: false, + cancellationToken: ct); + + using var demux = new StreamDemuxer(ws, StreamType.RemoteCommand); + demux.Start(); + + using var stdoutStream = demux.GetStream(ChannelIndex.StdOut, null); + using var stderrStream = demux.GetStream(ChannelIndex.StdErr, null); + // Channel 3 (Error) carries the terminal v1.Status payload with the real exit code. + using var statusStream = demux.GetStream(ChannelIndex.Error, null); + + var stdoutTask = ReadBoundedAsync(stdoutStream, maxOutputBytes, ct); + var stderrTask = ReadBoundedAsync(stderrStream, maxOutputBytes, ct); + var statusTask = ReadBoundedAsync(statusStream, maxOutputBytes, ct); + + await Task.WhenAll(stdoutTask, stderrTask, statusTask); + + var (stdoutBytes, stdoutTruncated) = await stdoutTask; + var (stderrBytes, stderrTruncated) = await stderrTask; + var (statusBytes, _) = await statusTask; + + var stdout = SandboxOutputRedactor.Default.Redact(Encoding.UTF8.GetString(stdoutBytes)); + var stderr = SandboxOutputRedactor.Default.Redact(Encoding.UTF8.GetString(stderrBytes)); + var exitCode = ParseExitCode(Encoding.UTF8.GetString(statusBytes)); + + return new SandboxExecResult( + exitCode, stdout, stderr, false, stdoutTruncated || stderrTruncated); + } + + /// + /// Reads up to from a stream, stopping at the cap. + /// Returns the bytes collected and whether the output was truncated. + /// + private static async Task<(byte[] Bytes, bool Truncated)> ReadBoundedAsync( + Stream stream, int maxBytes, CancellationToken ct) + { + using var buffer = new MemoryStream(); + var chunk = new byte[8192]; + bool truncated = false; + int read; + while ((read = await stream.ReadAsync(chunk, ct)) > 0) + { + int remaining = maxBytes - (int)buffer.Length; + if (remaining <= 0) { truncated = true; break; } + int take = Math.Min(read, remaining); + buffer.Write(chunk, 0, take); + if (take < read) { truncated = true; break; } + } + return (buffer.ToArray(), truncated); + } + + /// + /// Parses the terminal v1.Status JSON emitted on channel 3. + /// status: "Success" → exit 0. status: "Failure" → the ExitCode + /// cause from details.causes (defaulting to 1 if not present). + /// + private static int ParseExitCode(string statusJson) + { + if (string.IsNullOrWhiteSpace(statusJson)) + return 0; + + try + { + using var doc = JsonDocument.Parse(statusJson); + var root = doc.RootElement; + + var status = root.TryGetProperty("status", out var s) ? s.GetString() : null; + if (string.Equals(status, "Success", StringComparison.OrdinalIgnoreCase)) + return 0; + + if (root.TryGetProperty("details", out var details) && + details.TryGetProperty("causes", out var causes) && + causes.ValueKind == JsonValueKind.Array) + { + foreach (var cause in causes.EnumerateArray()) + { + var reason = cause.TryGetProperty("reason", out var r) ? r.GetString() : null; + if (string.Equals(reason, "ExitCode", StringComparison.OrdinalIgnoreCase) && + cause.TryGetProperty("message", out var m) && + int.TryParse(m.GetString(), out var code)) + return code; + } + } + + // Failure status with no parseable ExitCode cause → non-zero. + return 1; + } + catch (JsonException) + { + return 0; + } + } + + private string ResolvePodWorkingDirectory(string requestedWorkingDirectory) + { + var mountPath = NormalizeUnixPath(_options.WorkspaceMountPath, forceAbsolute: true); + if (string.IsNullOrWhiteSpace(requestedWorkingDirectory)) + return mountPath; + + var requested = NormalizeUnixPath(requestedWorkingDirectory, forceAbsolute: false); + if (IsSameOrChildPath(requested, mountPath)) + return requested; + + throw new InvalidOperationException( + $"Kubernetes sandbox working directory '{requestedWorkingDirectory}' is not under mounted workspace '{mountPath}'. " + + "Configure Workspace:PersistentVolume:MountRoot/Workspace:Path to match the workspace PVC mount used by sandbox pods."); + } + + private static bool IsSameOrChildPath(string path, string root) => + string.Equals(path, root, StringComparison.Ordinal) + || (root == "/" && path.StartsWith("/", StringComparison.Ordinal)) + || path.StartsWith(root + "/", StringComparison.Ordinal); + + private static string NormalizeUnixPath(string path, bool forceAbsolute) + { + var normalized = path.Trim().Replace('\\', '/'); + while (normalized.Contains("//", StringComparison.Ordinal)) + normalized = normalized.Replace("//", "/", StringComparison.Ordinal); + if (forceAbsolute && !normalized.StartsWith("/", StringComparison.Ordinal)) + normalized = "/" + normalized; + return normalized.Length > 1 ? normalized.TrimEnd('/') : normalized; + } + + private static string BuildShellScript(SandboxCommand command, string podWorkingDirectory) + { + var sb = new StringBuilder(); + + if (command.Environment is { Count: > 0 }) + { + foreach (var (key, value) in command.Environment) + sb.AppendLine($"export {key}={ShellSingleQuote(value)}"); + } + + sb.AppendLine($"cd {ShellSingleQuote(podWorkingDirectory)}"); + + sb.Append(command.CommandLine); + return sb.ToString(); + } + + private static string ShellSingleQuote(string s) => + "'" + s.Replace("'", "'\\''") + "'"; +} diff --git a/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs b/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs index 51265fe58..ccd1e294f 100644 --- a/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs +++ b/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs @@ -1,184 +1,184 @@ -using k8s; -using Agentweaver.SandboxExec; -using Agentweaver.AgentRuntime.Workflow; -using Agentweaver.Api.Infrastructure; -using Agentweaver.Domain; -using Microsoft.Extensions.Configuration; -using Microsoft.Extensions.Logging; - -namespace Agentweaver.Api.Sandbox; - -/// -/// Selects ISandboxExecutor based on: -/// 1. Sandbox:Backend config override ("kubernetes" or "local"). -/// 2. KUBERNETES_SERVICE_HOST environment variable (implicit in-cluster probe). -/// -/// Fail-closed: if running in-cluster and Kubernetes client initialization fails, -/// throws rather than silently falling back to a local executor. -/// -public sealed class SandboxExecutorRouter : ISandboxExecutorRouter -{ - private readonly IConfiguration _config; - private readonly ILoggerFactory _loggerFactory; - private readonly IPodNameRegistry? _podRegistry; - private readonly IAgentHostTurnTokenRegistry? _turnTokenRegistry; - private readonly Security.IRunAuthorshipCapabilityStore? _authorshipCapabilityStore; - private readonly IHttpClientFactory? _httpClientFactory; - private readonly IRunSubmittingUserResolver? _submittingUserResolver; - private readonly IGitHubTokenStore? _tokenStore; - private readonly IGitHubTokenScopeProvider? _tokenScopeProvider; - private readonly Agentweaver.Api.Auth.ISecretStore? _secretStore; - private readonly IRunEventStream? _runEventStream; - private readonly IRunOptionsStore? _runOptions; - private readonly IGitHubAccessTokenProvider? _accessTokenProvider; - private readonly Preview.ISandboxPreviewService? _previewService; - private readonly RunRepositoryCredentialRegistry? _repositoryCredentials; - - public SandboxExecutorRouter(IConfiguration config, ILoggerFactory loggerFactory, - IPodNameRegistry? podRegistry = null, IHttpClientFactory? httpClientFactory = null, - IRunSubmittingUserResolver? submittingUserResolver = null, - IAgentHostTurnTokenRegistry? turnTokenRegistry = null, - IGitHubTokenStore? tokenStore = null, - IGitHubTokenScopeProvider? tokenScopeProvider = null, - Agentweaver.Api.Auth.ISecretStore? secretStore = null, - IRunEventStream? runEventStream = null, - IRunOptionsStore? runOptions = null, - IGitHubAccessTokenProvider? accessTokenProvider = null, - Preview.ISandboxPreviewService? previewService = null, - RunRepositoryCredentialRegistry? repositoryCredentials = null, - Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null) - { - _config = config; - _loggerFactory = loggerFactory; - _podRegistry = podRegistry; - _turnTokenRegistry = turnTokenRegistry; - _httpClientFactory = httpClientFactory; - _submittingUserResolver = submittingUserResolver; - _tokenStore = tokenStore; - _tokenScopeProvider = tokenScopeProvider; - _secretStore = secretStore; - _runEventStream = runEventStream; - _runOptions = runOptions; - _accessTokenProvider = accessTokenProvider; - _previewService = previewService; - _repositoryCredentials = repositoryCredentials; - _authorshipCapabilityStore = authorshipCapabilityStore; - } - - public ISandboxExecutor Resolve() - { - var backendOverride = _config["Sandbox:Backend"]?.ToLowerInvariant(); - var isInCluster = SandboxExecutorFactory.IsInCluster; - var logger = _loggerFactory.CreateLogger(); - - var useKubernetes = backendOverride == "kubernetes" - || (isInCluster && backendOverride != "local"); - - if (!useKubernetes) - { - logger.LogInformation( - "SandboxExecutorRouter: selecting local executor (backend={Backend}, inCluster={InCluster})", - backendOverride ?? "(none)", isInCluster); - var localExecutor = SandboxExecutorFactory.Create(logger); - if (!localExecutor.IsRealIsolation) - { - logger.LogWarning( - "⚠️ PassthroughExecutor selected — agent commands run directly on the host. Not for production use."); - } - return localExecutor; - } - - try - { - var k8sConfig = KubernetesClientConfiguration.InClusterConfig(); - var k8sClient = new Kubernetes(k8sConfig); - var sandboxOptions = new KubernetesSandboxOptions - { - Namespace = _config["Sandbox:Kubernetes:Namespace"] ?? "agentweaver", - TemplateRef = _config["Sandbox:Kubernetes:TemplateRef"] ?? "agentweaver-sandbox", - WarmPoolRef = _config["Sandbox:Kubernetes:WarmPoolRef"] - ?? _config["Sandbox:Kubernetes:TemplateRef"] ?? "agentweaver-sandbox", - AgentHostWarmPoolRef = _config["Sandbox:Kubernetes:AgentHostWarmPoolRef"] - ?? "agentweaver-agent-host", - WorkspaceMountPath = _config["Sandbox:Kubernetes:WorkspaceMountPath"] - ?? _config["Workspace:PersistentVolume:MountRoot"] - ?? _config["Workspace:Path"] - ?? "/workspace", - TimeoutSeconds = int.TryParse( - _config["Sandbox:Kubernetes:TimeoutSeconds"], out int t) ? t : 600, - ServiceCidr = _config["Sandbox:Kubernetes:ServiceCidr"] - ?? _config["Sandbox:Kubernetes:ClusterServiceCidr"], - SandboxEgressCidrExclusions = ReadSandboxEgressCidrExclusions(), - RequireMtls = !string.Equals( - _config["Sandbox:AgentHost:RequireMtls"], "false", StringComparison.OrdinalIgnoreCase), - AgentHostHealthzPath = _config["Sandbox:Kubernetes:AgentHostHealthzPath"] ?? "/healthz", - AgentHostReadyTimeoutSeconds = int.TryParse( - _config["Sandbox:Kubernetes:AgentHostReadyTimeoutSeconds"], out int rt) ? rt : 90, - AgentHostReadyPollIntervalMs = int.TryParse( - _config["Sandbox:Kubernetes:AgentHostReadyPollIntervalMs"], out int ri) ? ri : 1000, - // Option C warm-pool token fetch: same KV the API persists user tokens to. - KvUri = _config["Sandbox:AgentHost:KeyVaultUri"] - ?? _config["Auth:TokenStore:KeyVaultUri"], - }; - var k8sLogger = _loggerFactory.CreateLogger(); - WarnIfServiceCidrNotExcluded(sandboxOptions, logger); - - // Readiness gate closes the A2A cold-start race (pod Running before Kestrel binds :8088). - // Requires the named HttpClient that can reach the pod IP; skipped (null) only if no - // IHttpClientFactory was injected (which would itself be a misconfiguration in-cluster). - IAgentHostReadinessProbe? readinessProbe = null; - if (_httpClientFactory is not null) - { - readinessProbe = new HttpAgentHostReadinessProbe( - _httpClientFactory, - TimeSpan.FromSeconds(sandboxOptions.AgentHostReadyTimeoutSeconds), - TimeSpan.FromMilliseconds(sandboxOptions.AgentHostReadyPollIntervalMs), - _loggerFactory.CreateLogger()); - } - else - { - logger.LogWarning( - "SandboxExecutorRouter: no IHttpClientFactory available — AgentHost readiness gate disabled. " + - "First A2A turns may race the cold-start Kestrel bind."); - } - - logger.LogInformation( - "SandboxExecutorRouter: selecting KubernetesSandboxExecutor (namespace={Namespace}, workspaceMountPath={WorkspaceMountPath})", - sandboxOptions.Namespace, sandboxOptions.WorkspaceMountPath); - return new KubernetesSandboxExecutor( - k8sClient, sandboxOptions, k8sLogger, _podRegistry, _turnTokenRegistry, readinessProbe, - _submittingUserResolver, _httpClientFactory, _tokenStore, _secretStore, _runEventStream, - _runOptions, _repositoryCredentials, _accessTokenProvider, _previewService, - tokenScopeProvider: _tokenScopeProvider, - authorshipCapabilityStore: _authorshipCapabilityStore); - } - catch (Exception ex) - { - throw new InvalidOperationException( - "SandboxExecutorRouter: in-cluster Kubernetes executor initialization failed. " + - "Fail-closed: will not fall back to a local executor.", ex); - } - } - - private IReadOnlyList ReadSandboxEgressCidrExclusions() => - _config.GetSection("Sandbox:Kubernetes:SandboxEgressCidrExclusions").Get() - ?? _config.GetSection("SandboxEgressCidrExclusions").Get() - ?? []; - - private static void WarnIfServiceCidrNotExcluded( - KubernetesSandboxOptions options, - ILogger logger) - { - if (string.IsNullOrWhiteSpace(options.ServiceCidr)) - return; - - var excluded = options.SandboxEgressCidrExclusions.Any(cidr => - string.Equals(cidr.Trim(), options.ServiceCidr.Trim(), StringComparison.OrdinalIgnoreCase)); - if (!excluded) - { - logger.LogWarning( - "Sandbox egress configuration warning: cluster service CIDR {ServiceCidr} is not listed in SandboxEgressCidrExclusions. Add it to keep sandbox egress from reaching in-cluster services.", - options.ServiceCidr); - } - } -} +using k8s; +using Agentweaver.SandboxExec; +using Agentweaver.AgentRuntime.Workflow; +using Agentweaver.Api.Infrastructure; +using Agentweaver.Domain; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Logging; + +namespace Agentweaver.Api.Sandbox; + +/// +/// Selects ISandboxExecutor based on: +/// 1. Sandbox:Backend config override ("kubernetes" or "local"). +/// 2. KUBERNETES_SERVICE_HOST environment variable (implicit in-cluster probe). +/// +/// Fail-closed: if running in-cluster and Kubernetes client initialization fails, +/// throws rather than silently falling back to a local executor. +/// +public sealed class SandboxExecutorRouter : ISandboxExecutorRouter +{ + private readonly IConfiguration _config; + private readonly ILoggerFactory _loggerFactory; + private readonly IPodNameRegistry? _podRegistry; + private readonly IAgentHostTurnTokenRegistry? _turnTokenRegistry; + private readonly Security.IRunAuthorshipCapabilityStore? _authorshipCapabilityStore; + private readonly IHttpClientFactory? _httpClientFactory; + private readonly IRunSubmittingUserResolver? _submittingUserResolver; + private readonly IGitHubTokenStore? _tokenStore; + private readonly IGitHubTokenScopeProvider? _tokenScopeProvider; + private readonly Agentweaver.Api.Auth.ISecretStore? _secretStore; + private readonly IRunEventStream? _runEventStream; + private readonly IRunOptionsStore? _runOptions; + private readonly IGitHubAccessTokenProvider? _accessTokenProvider; + private readonly Preview.ISandboxPreviewService? _previewService; + private readonly RunRepositoryCredentialRegistry? _repositoryCredentials; + + public SandboxExecutorRouter(IConfiguration config, ILoggerFactory loggerFactory, + IPodNameRegistry? podRegistry = null, IHttpClientFactory? httpClientFactory = null, + IRunSubmittingUserResolver? submittingUserResolver = null, + IAgentHostTurnTokenRegistry? turnTokenRegistry = null, + IGitHubTokenStore? tokenStore = null, + IGitHubTokenScopeProvider? tokenScopeProvider = null, + Agentweaver.Api.Auth.ISecretStore? secretStore = null, + IRunEventStream? runEventStream = null, + IRunOptionsStore? runOptions = null, + IGitHubAccessTokenProvider? accessTokenProvider = null, + Preview.ISandboxPreviewService? previewService = null, + RunRepositoryCredentialRegistry? repositoryCredentials = null, + Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null) + { + _config = config; + _loggerFactory = loggerFactory; + _podRegistry = podRegistry; + _turnTokenRegistry = turnTokenRegistry; + _httpClientFactory = httpClientFactory; + _submittingUserResolver = submittingUserResolver; + _tokenStore = tokenStore; + _tokenScopeProvider = tokenScopeProvider; + _secretStore = secretStore; + _runEventStream = runEventStream; + _runOptions = runOptions; + _accessTokenProvider = accessTokenProvider; + _previewService = previewService; + _repositoryCredentials = repositoryCredentials; + _authorshipCapabilityStore = authorshipCapabilityStore; + } + + public ISandboxExecutor Resolve() + { + var backendOverride = _config["Sandbox:Backend"]?.ToLowerInvariant(); + var isInCluster = SandboxExecutorFactory.IsInCluster; + var logger = _loggerFactory.CreateLogger(); + + var useKubernetes = backendOverride == "kubernetes" + || (isInCluster && backendOverride != "local"); + + if (!useKubernetes) + { + logger.LogInformation( + "SandboxExecutorRouter: selecting local executor (backend={Backend}, inCluster={InCluster})", + backendOverride ?? "(none)", isInCluster); + var localExecutor = SandboxExecutorFactory.Create(logger); + if (!localExecutor.IsRealIsolation) + { + logger.LogWarning( + "⚠️ PassthroughExecutor selected — agent commands run directly on the host. Not for production use."); + } + return localExecutor; + } + + try + { + var k8sConfig = KubernetesClientConfiguration.InClusterConfig(); + var k8sClient = new Kubernetes(k8sConfig); + var sandboxOptions = new KubernetesSandboxOptions + { + Namespace = _config["Sandbox:Kubernetes:Namespace"] ?? "agentweaver", + TemplateRef = _config["Sandbox:Kubernetes:TemplateRef"] ?? "agentweaver-sandbox", + WarmPoolRef = _config["Sandbox:Kubernetes:WarmPoolRef"] + ?? _config["Sandbox:Kubernetes:TemplateRef"] ?? "agentweaver-sandbox", + AgentHostWarmPoolRef = _config["Sandbox:Kubernetes:AgentHostWarmPoolRef"] + ?? "agentweaver-agent-host", + WorkspaceMountPath = _config["Sandbox:Kubernetes:WorkspaceMountPath"] + ?? _config["Workspace:PersistentVolume:MountRoot"] + ?? _config["Workspace:Path"] + ?? "/workspace", + TimeoutSeconds = int.TryParse( + _config["Sandbox:Kubernetes:TimeoutSeconds"], out int t) ? t : 600, + ServiceCidr = _config["Sandbox:Kubernetes:ServiceCidr"] + ?? _config["Sandbox:Kubernetes:ClusterServiceCidr"], + SandboxEgressCidrExclusions = ReadSandboxEgressCidrExclusions(), + RequireMtls = !string.Equals( + _config["Sandbox:AgentHost:RequireMtls"], "false", StringComparison.OrdinalIgnoreCase), + AgentHostHealthzPath = _config["Sandbox:Kubernetes:AgentHostHealthzPath"] ?? "/healthz", + AgentHostReadyTimeoutSeconds = int.TryParse( + _config["Sandbox:Kubernetes:AgentHostReadyTimeoutSeconds"], out int rt) ? rt : 90, + AgentHostReadyPollIntervalMs = int.TryParse( + _config["Sandbox:Kubernetes:AgentHostReadyPollIntervalMs"], out int ri) ? ri : 1000, + // Option C warm-pool token fetch: same KV the API persists user tokens to. + KvUri = _config["Sandbox:AgentHost:KeyVaultUri"] + ?? _config["Auth:TokenStore:KeyVaultUri"], + }; + var k8sLogger = _loggerFactory.CreateLogger(); + WarnIfServiceCidrNotExcluded(sandboxOptions, logger); + + // Readiness gate closes the A2A cold-start race (pod Running before Kestrel binds :8088). + // Requires the named HttpClient that can reach the pod IP; skipped (null) only if no + // IHttpClientFactory was injected (which would itself be a misconfiguration in-cluster). + IAgentHostReadinessProbe? readinessProbe = null; + if (_httpClientFactory is not null) + { + readinessProbe = new HttpAgentHostReadinessProbe( + _httpClientFactory, + TimeSpan.FromSeconds(sandboxOptions.AgentHostReadyTimeoutSeconds), + TimeSpan.FromMilliseconds(sandboxOptions.AgentHostReadyPollIntervalMs), + _loggerFactory.CreateLogger()); + } + else + { + logger.LogWarning( + "SandboxExecutorRouter: no IHttpClientFactory available — AgentHost readiness gate disabled. " + + "First A2A turns may race the cold-start Kestrel bind."); + } + + logger.LogInformation( + "SandboxExecutorRouter: selecting KubernetesSandboxExecutor (namespace={Namespace}, workspaceMountPath={WorkspaceMountPath})", + sandboxOptions.Namespace, sandboxOptions.WorkspaceMountPath); + return new KubernetesSandboxExecutor( + k8sClient, sandboxOptions, k8sLogger, _podRegistry, _turnTokenRegistry, readinessProbe, + _submittingUserResolver, _httpClientFactory, _tokenStore, _secretStore, _runEventStream, + _runOptions, _repositoryCredentials, _accessTokenProvider, _previewService, + tokenScopeProvider: _tokenScopeProvider, + authorshipCapabilityStore: _authorshipCapabilityStore); + } + catch (Exception ex) + { + throw new InvalidOperationException( + "SandboxExecutorRouter: in-cluster Kubernetes executor initialization failed. " + + "Fail-closed: will not fall back to a local executor.", ex); + } + } + + private IReadOnlyList ReadSandboxEgressCidrExclusions() => + _config.GetSection("Sandbox:Kubernetes:SandboxEgressCidrExclusions").Get() + ?? _config.GetSection("SandboxEgressCidrExclusions").Get() + ?? []; + + private static void WarnIfServiceCidrNotExcluded( + KubernetesSandboxOptions options, + ILogger logger) + { + if (string.IsNullOrWhiteSpace(options.ServiceCidr)) + return; + + var excluded = options.SandboxEgressCidrExclusions.Any(cidr => + string.Equals(cidr.Trim(), options.ServiceCidr.Trim(), StringComparison.OrdinalIgnoreCase)); + if (!excluded) + { + logger.LogWarning( + "Sandbox egress configuration warning: cluster service CIDR {ServiceCidr} is not listed in SandboxEgressCidrExclusions. Add it to keep sandbox egress from reaching in-cluster services.", + options.ServiceCidr); + } + } +} diff --git a/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs b/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs index afc82cb04..ca32f8e90 100644 --- a/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs +++ b/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs @@ -1,571 +1,571 @@ -using System.Net.Http.Headers; -using System.Net.Http.Json; -using System.Security.Cryptography; -using System.Text; -using System.Text.Json; -using Agentweaver.Api.Auth; -using Agentweaver.Api.Memory; -using Microsoft.EntityFrameworkCore; -using Microsoft.IdentityModel.JsonWebTokens; -using Microsoft.IdentityModel.Tokens; - -namespace Agentweaver.Api.Webhooks; - -public enum RepoAppInstallationOutcome { Success, InstallationUnavailable, ConfigurationUnavailable, ProviderUnavailable } -internal enum RepoAppInstallationBindingOutcome { Bound, PermissionChanged, Conflict } - -internal sealed record RepoAppInstallationAuthority( - long InstallationId, - long RepositoryId, - string FullNameDisplay, - IReadOnlyDictionary Permissions); -internal sealed record RepoAppInstallationToken(string Value, DateTimeOffset? ExpiresAt); - -/// -/// API-only boundary for a short-lived Repo App JWT and the single-repository installation -/// token it mints. Neither credential is written to persistence, logs, or HTTP responses. -/// -public sealed class RepoAppInstallationTokenService( - IConfiguration configuration, - MemoryDbContext db, - ISecretStore secretStore, - IHttpClientFactory httpClientFactory) -{ - private static readonly TimeSpan JwtLifetime = TimeSpan.FromMinutes(9); - private static readonly IReadOnlyDictionary UnattendedRepositoryPermissionCeilings = - new Dictionary(StringComparer.Ordinal) - { - ["contents"] = "write", - ["pull_requests"] = "write", - }; - private static readonly IReadOnlyDictionary RepositoryMetadataPermissionScope = - new Dictionary(StringComparer.Ordinal) - { - ["metadata"] = "read", - }; - - public async Task MintForRepositoryAsync( - long installationId, - long repositoryId, - Func useToken, - CancellationToken ct = default) - { - if (installationId <= 0 || repositoryId <= 0) - return RepoAppInstallationOutcome.InstallationUnavailable; - - var installationActive = await db.GitHubInstallations.AsNoTracking() - .AnyAsync(x => x.InstallationId == installationId && - x.AppKind == GitHubAppKind.Repo && - x.RevokedAt == null, ct).ConfigureAwait(false); - var grant = await db.GitHubRepositoryGrants.AsNoTracking() - .SingleOrDefaultAsync(x => x.InstallationId == installationId && - x.RepositoryId == repositoryId && - x.RevokedAt == null, ct).ConfigureAwait(false); - if (!installationActive || grant is null) - return RepoAppInstallationOutcome.InstallationUnavailable; - - var authority = await GetRepositoryAuthorityAsync(installationId, repositoryId, ct).ConfigureAwait(false); - if (authority is null) - return RepoAppInstallationOutcome.ProviderUnavailable; - if (!CryptographicOperations.FixedTimeEquals( - Encoding.UTF8.GetBytes(grant.PermissionDigest), - Encoding.UTF8.GetBytes(CreatePermissionDigest(authority.Permissions)))) - { - await new RepoAppInstallationLifecycleService(db) - .InvalidateForPermissionChangeAsync(installationId, repositoryId, ct).ConfigureAwait(false); - return RepoAppInstallationOutcome.InstallationUnavailable; - } - if (!TryCreateUnattendedPermissionScope(authority.Permissions, out var requestedPermissions)) - return RepoAppInstallationOutcome.InstallationUnavailable; - - try - { - var appJwt = await CreateAppJwtAsync(ct).ConfigureAwait(false); - if (appJwt is null) - return RepoAppInstallationOutcome.ConfigurationUnavailable; - var installationToken = await GetInstallationTokenAsync( - appJwt, installationId, repositoryId, requestedPermissions, ct).ConfigureAwait(false); - if (installationToken is null) - return RepoAppInstallationOutcome.ProviderUnavailable; - - if (installationToken.ExpiresAt is null || installationToken.ExpiresAt <= DateTimeOffset.UtcNow) - return RepoAppInstallationOutcome.ProviderUnavailable; - await useToken(installationToken.Value, installationToken.ExpiresAt.Value).ConfigureAwait(false); - return RepoAppInstallationOutcome.Success; - } - catch (OperationCanceledException) when (!ct.IsCancellationRequested) - { - return RepoAppInstallationOutcome.ProviderUnavailable; - } - catch (HttpRequestException) - { - return RepoAppInstallationOutcome.ProviderUnavailable; - } - } - - public async Task VerifyRepositoryInstallationAsync( - long installationId, - long repositoryId, - CancellationToken ct = default) - => await GetRepositoryAuthorityAsync(installationId, repositoryId, ct).ConfigureAwait(false) is not null; - - /// Revokes a minted installation credential. This method does not persist or log it. - public async Task RevokeRepositoryTokenAsync(string token, CancellationToken ct = default) - { - if (string.IsNullOrWhiteSpace(token)) - return; - - try - { - using var request = CreateGitHubRequest(HttpMethod.Delete, "/installation/token", token); - using var response = await httpClientFactory.CreateClient("github").SendAsync(request, ct) - .ConfigureAwait(false); - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) - { - throw; - } - catch (HttpRequestException) - { - // Token expiry is the backstop. Release and orphan cleanup must not fail on revoke. - } - } - - /// - /// Resolves the installation's exact repository authority from GitHub. The request supplies - /// only numeric identifiers; permissions and the display name are provider-owned values. - /// - internal async Task GetRepositoryAuthorityAsync( - long installationId, - long repositoryId, - CancellationToken ct = default) - { - if (installationId <= 0 || repositoryId <= 0) - return null; - - var appJwt = await CreateAppJwtAsync(ct).ConfigureAwait(false); - if (appJwt is null) - return null; - - try - { - using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); - timeout.CancelAfter(TimeSpan.FromSeconds(10)); - var client = httpClientFactory.CreateClient("github"); - using var installationRequest = CreateGitHubRequest( - HttpMethod.Get, $"/repositories/{repositoryId}/installation", appJwt); - using var installationResponse = await client.SendAsync(installationRequest, timeout.Token).ConfigureAwait(false); - if (!installationResponse.IsSuccessStatusCode) - return null; - using var installationDocument = JsonDocument.Parse( - await installationResponse.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); - var installation = installationDocument.RootElement; - if (!installation.TryGetProperty("id", out var actualInstallation) || - !actualInstallation.TryGetInt64(out var actualInstallationId) || - actualInstallationId != installationId || - !installation.TryGetProperty("repository_selection", out var repositorySelection) || - repositorySelection.ValueKind != JsonValueKind.String || - string.IsNullOrWhiteSpace(repositorySelection.GetString()) || - !installation.TryGetProperty("account", out var account) || - account.ValueKind != JsonValueKind.Object || - !TryGetNormalizedPermissions(installation, out var permissions)) - return null; - - var metadataToken = await GetInstallationTokenAsync( - appJwt, installationId, repositoryId, RepositoryMetadataPermissionScope, timeout.Token) - .ConfigureAwait(false); - if (metadataToken is null) - return null; - using var repositoryRequest = CreateGitHubRequest( - HttpMethod.Get, $"/repositories/{repositoryId}", metadataToken.Value); - using var repositoryResponse = await client.SendAsync(repositoryRequest, timeout.Token).ConfigureAwait(false); - if (!repositoryResponse.IsSuccessStatusCode) - return null; - using var repositoryDocument = JsonDocument.Parse( - await repositoryResponse.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); - var repository = repositoryDocument.RootElement; - if (!repository.TryGetProperty("id", out var actualRepository) || - !actualRepository.TryGetInt64(out var actualRepositoryId) || - actualRepositoryId != repositoryId || - !repository.TryGetProperty("full_name", out var fullName) || - fullName.ValueKind != JsonValueKind.String || - string.IsNullOrWhiteSpace(fullName.GetString())) - return null; - - return new RepoAppInstallationAuthority( - installationId, repositoryId, fullName.GetString()!, permissions); - } - catch (OperationCanceledException) when (!ct.IsCancellationRequested) - { - return null; - } - catch (HttpRequestException) - { - return null; - } - catch (JsonException) - { - return null; - } - } - - private async Task CreateAppJwtAsync(CancellationToken ct) - { - if (!long.TryParse(configuration["Auth:RepoApp:AppId"], out var appId) || appId <= 0 || - string.IsNullOrWhiteSpace(configuration["Auth:RepoApp:PrivateKeySecretName"])) - return null; - var pem = await secretStore.GetSecretAsync(configuration["Auth:RepoApp:PrivateKeySecretName"]!, ct) - .ConfigureAwait(false); - if (!pem.Found || string.IsNullOrWhiteSpace(pem.Value)) - return null; - try - { - return CreateAppJwt(appId, pem.Value); - } - catch (CryptographicException) - { - return null; - } - } - - private HttpRequestMessage CreateGitHubRequest(HttpMethod method, string path, string appJwt) - { - var request = new HttpRequestMessage( - method, $"{(configuration["Auth:RepoApp:ApiUrl"] ?? "https://api.github.com").TrimEnd('/')}{path}"); - request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", appJwt); - request.Headers.UserAgent.ParseAdd("Agentweaver/1.0"); - request.Headers.Accept.ParseAdd("application/vnd.github+json"); - return request; - } - - private async Task GetInstallationTokenAsync( - string appJwt, - long installationId, - long repositoryId, - IReadOnlyDictionary permissions, - CancellationToken ct) - { - using var request = CreateGitHubRequest( - HttpMethod.Post, $"/app/installations/{installationId}/access_tokens", appJwt); - request.Content = JsonContent.Create(new { repository_ids = new[] { repositoryId }, permissions }); - using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); - timeout.CancelAfter(TimeSpan.FromSeconds(10)); - using var response = await httpClientFactory.CreateClient("github").SendAsync(request, timeout.Token) - .ConfigureAwait(false); - if (!response.IsSuccessStatusCode) - return null; - using var document = JsonDocument.Parse( - await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); - if (!document.RootElement.TryGetProperty("token", out var token) || - string.IsNullOrWhiteSpace(token.GetString())) - return null; - DateTimeOffset? expiresAt = document.RootElement.TryGetProperty("expires_at", out var expiresAtElement) && - DateTimeOffset.TryParse(expiresAtElement.GetString(), out var parsedExpiry) - ? parsedExpiry - : null; - return new(token.GetString()!, expiresAt); - } - - private static bool TryGetNormalizedPermissions( - JsonElement installation, - out IReadOnlyDictionary permissions) - { - permissions = new Dictionary(); - if (!installation.TryGetProperty("permissions", out var source) || - source.ValueKind != JsonValueKind.Object) - return false; - - var normalized = new Dictionary(StringComparer.Ordinal); - foreach (var permission in source.EnumerateObject()) - { - if (permission.Value.ValueKind != JsonValueKind.String) - return false; - var name = permission.Name.Trim().ToLowerInvariant(); - var value = permission.Value.GetString()?.Trim().ToLowerInvariant(); - if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(value) || - !normalized.TryAdd(name, value)) - return false; - } - permissions = normalized; - return normalized.Count > 0; - } - - private static bool TryCreateUnattendedPermissionScope( - IReadOnlyDictionary providerPermissions, - out IReadOnlyDictionary requestedPermissions) - { - var requested = new Dictionary(StringComparer.Ordinal); - foreach (var ceiling in UnattendedRepositoryPermissionCeilings) - { - if (!providerPermissions.TryGetValue(ceiling.Key, out var actual)) - continue; - if (!string.Equals(actual, "read", StringComparison.Ordinal) && - !string.Equals(actual, "write", StringComparison.Ordinal)) - { - requestedPermissions = new Dictionary(); - return false; - } - if (string.Equals(ceiling.Value, "read", StringComparison.Ordinal) && - string.Equals(actual, "write", StringComparison.Ordinal)) - { - requestedPermissions = new Dictionary(); - return false; - } - requested[ceiling.Key] = actual; - } - requestedPermissions = requested; - return requested.Count > 0; - } - - internal static string CreateAppJwt(long appId, string pem) - { - using var rsa = RSA.Create(); - rsa.ImportFromPem(pem); - var now = DateTime.UtcNow; - var signingKey = new RsaSecurityKey(rsa) - { - CryptoProviderFactory = new CryptoProviderFactory { CacheSignatureProviders = false }, - }; - return new JsonWebTokenHandler().CreateToken(new SecurityTokenDescriptor - { - Issuer = appId.ToString(System.Globalization.CultureInfo.InvariantCulture), - IssuedAt = now.AddMinutes(-1), - NotBefore = now.AddMinutes(-1), - Expires = now.Add(JwtLifetime), - SigningCredentials = new SigningCredentials(signingKey, SecurityAlgorithms.RsaSha256), - }); - } - - internal static string CreatePermissionDigest(IReadOnlyDictionary permissions) - { - var canonical = string.Join("&", permissions.OrderBy(x => x.Key, StringComparer.Ordinal) - .Select(x => $"{x.Key.Trim().ToLowerInvariant()}={x.Value.Trim().ToLowerInvariant()}")); - return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(canonical))).ToLowerInvariant(); - } -} - -/// Durable installation/grant state machine for authenticated Repo App deliveries. -public sealed class RepoAppInstallationLifecycleService(MemoryDbContext db) -{ - private const string CompletedEventPrefix = "completed/"; - private static readonly TimeSpan ProcessingLease = TimeSpan.FromMinutes(10); - - public async Task<(bool Claimed, IReadOnlyList ProjectIds)> ProcessAsync( - string deliveryId, - string eventName, - GitHubWebhookPayload payload, - CancellationToken ct = default) - { - if (string.IsNullOrWhiteSpace(deliveryId)) - return (false, []); - - db.ChangeTracker.Clear(); - await using var transaction = await db.Database.BeginTransactionAsync(ct).ConfigureAwait(false); - db.GitHubLifecycleDeliveries.Add(new GitHubLifecycleDeliveryRecord - { - DeliveryId = deliveryId, - EventName = eventName, - InstallationId = payload.Installation?.Id, - RepositoryId = payload.Repository?.Id, - ReceivedAt = DateTimeOffset.UtcNow, - }); - try - { - await db.SaveChangesAsync(ct).ConfigureAwait(false); - } - catch (DbUpdateException) - { - await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); - db.ChangeTracker.Clear(); - var leaseExpiresBefore = DateTimeOffset.UtcNow.Subtract(ProcessingLease); - var abandoned = await db.GitHubLifecycleDeliveries.FindAsync([deliveryId], ct).ConfigureAwait(false); - if (abandoned is null || abandoned.EventName != eventName || abandoned.ReceivedAt >= leaseExpiresBefore) - return (false, []); - var reclaimed = await db.GitHubLifecycleDeliveries - .Where(x => x.DeliveryId == deliveryId && - x.EventName == abandoned.EventName && - x.ReceivedAt == abandoned.ReceivedAt) - .ExecuteDeleteAsync(ct).ConfigureAwait(false); - if (reclaimed != 1) - return (false, []); - return await ProcessAsync(deliveryId, eventName, payload, ct).ConfigureAwait(false); - } - - var installationId = (payload.Installation?.Id).GetValueOrDefault(); - if (installationId > 0 && eventName is "installation" or "installation_repositories") - { - await ApplyLifecycleAsync(installationId, payload, ct).ConfigureAwait(false); - await db.SaveChangesAsync(ct).ConfigureAwait(false); - } - - db.ChangeTracker.Clear(); - var installationActive = installationId > 0 && await db.GitHubInstallations.AsNoTracking() - .AnyAsync(x => x.InstallationId == installationId && - x.AppKind == GitHubAppKind.Repo && - x.RevokedAt == null, ct).ConfigureAwait(false); - var projectIds = installationActive && payload.Repository?.Id is > 0 - ? await db.GitHubRepositoryGrants.AsNoTracking() - .Where(x => x.InstallationId == installationId && - x.RepositoryId == payload.Repository.Id && - x.RevokedAt == null) - .Select(x => x.ProjectId).ToListAsync(ct).ConfigureAwait(false) - : []; - await db.SaveChangesAsync(ct).ConfigureAwait(false); - await transaction.CommitAsync(ct).ConfigureAwait(false); - return (true, projectIds); - } - - /// - /// Releases a claim only when downstream dispatch did not complete, allowing GitHub to retry. - /// The dispatch path has its own delivery-id idempotency guard. - /// - public async Task ReleaseAsync(string deliveryId, CancellationToken ct = default) - { - db.ChangeTracker.Clear(); - await db.GitHubLifecycleDeliveries.Where(x => x.DeliveryId == deliveryId) - .ExecuteDeleteAsync(ct).ConfigureAwait(false); - } - - public Task IsCompletedAsync(string deliveryId, CancellationToken ct = default) => - db.GitHubLifecycleDeliveries.AsNoTracking() - .AnyAsync(x => x.DeliveryId == deliveryId && - x.EventName.StartsWith(CompletedEventPrefix), ct); - - public async Task CompleteAsync(string deliveryId, CancellationToken ct = default) - { - db.ChangeTracker.Clear(); - var current = await db.GitHubLifecycleDeliveries.AsNoTracking() - .Where(x => x.DeliveryId == deliveryId) - .Select(x => x.EventName).SingleOrDefaultAsync(ct).ConfigureAwait(false); - if (current is null) - return false; - if (current.StartsWith(CompletedEventPrefix, StringComparison.Ordinal)) - return true; - return await db.GitHubLifecycleDeliveries.Where(x => x.DeliveryId == deliveryId && x.EventName == current) - .ExecuteUpdateAsync(s => s.SetProperty(x => x.EventName, $"{CompletedEventPrefix}{current}"), ct) - .ConfigureAwait(false) == 1; - } - - internal async Task BindAsync( - string projectId, - RepoAppInstallationAuthority authority, - CancellationToken ct = default) - { - await using var transaction = await db.Database.BeginTransactionAsync( - System.Data.IsolationLevel.Serializable, ct).ConfigureAwait(false); - var now = DateTimeOffset.UtcNow; - var installation = await db.GitHubInstallations.FindAsync([authority.InstallationId], ct).ConfigureAwait(false); - if (installation is not null && installation.ProjectId is not null && - !string.Equals(installation.ProjectId, projectId, StringComparison.Ordinal)) - return RepoAppInstallationBindingOutcome.Conflict; - if (installation is null) - db.GitHubInstallations.Add(new GitHubInstallationRecord - { - InstallationId = authority.InstallationId, AppKind = GitHubAppKind.Repo, ProjectId = projectId, CreatedAt = now, - }); - else - { - installation.ProjectId = projectId; - installation.RevokedAt = null; - } - - var grant = await db.GitHubRepositoryGrants.FindAsync( - [authority.InstallationId, authority.RepositoryId], ct).ConfigureAwait(false); - if (grant is not null && !string.Equals(grant.ProjectId, projectId, StringComparison.Ordinal)) - return RepoAppInstallationBindingOutcome.Conflict; - if (grant is null) - db.GitHubRepositoryGrants.Add(new GitHubRepositoryGrantRecord - { - InstallationId = authority.InstallationId, RepositoryId = authority.RepositoryId, ProjectId = projectId, - FullNameDisplay = authority.FullNameDisplay, - PermissionDigest = RepoAppInstallationTokenService.CreatePermissionDigest(authority.Permissions), - GrantedAt = now, - }); - else - { - var permissionDigest = RepoAppInstallationTokenService.CreatePermissionDigest(authority.Permissions); - if (!CryptographicOperations.FixedTimeEquals( - Encoding.UTF8.GetBytes(grant.PermissionDigest), Encoding.UTF8.GetBytes(permissionDigest))) - { - grant.FullNameDisplay = authority.FullNameDisplay; - grant.RevokedAt = now; - await InvalidateForPermissionChangeAsync(authority.InstallationId, authority.RepositoryId, ct) - .ConfigureAwait(false); - await db.SaveChangesAsync(ct).ConfigureAwait(false); - await transaction.CommitAsync(ct).ConfigureAwait(false); - return RepoAppInstallationBindingOutcome.PermissionChanged; - } - grant.FullNameDisplay = authority.FullNameDisplay; - grant.RevokedAt = null; - } - try - { - await db.SaveChangesAsync(ct).ConfigureAwait(false); - await transaction.CommitAsync(ct).ConfigureAwait(false); - return RepoAppInstallationBindingOutcome.Bound; - } - catch (DbUpdateException) - { - db.ChangeTracker.Clear(); - return RepoAppInstallationBindingOutcome.Conflict; - } - } - - public async Task InvalidateForPermissionChangeAsync( - long installationId, - long repositoryId, - CancellationToken ct = default) - { - var transaction = db.Database.CurrentTransaction is null - ? await db.Database.BeginTransactionAsync(ct).ConfigureAwait(false) - : null; - var now = DateTimeOffset.UtcNow; - try - { - await db.GitHubRepositoryGrants - .Where(x => x.InstallationId == installationId && - x.RepositoryId == repositoryId && - x.RevokedAt == null) - .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); - await db.AutomationActivations - .Where(x => x.InstallationId == installationId && - x.RepositoryId == repositoryId && - x.Status != AutomationActivationStatus.Invalidated) - .ExecuteUpdateAsync(s => s - .SetProperty(x => x.Status, AutomationActivationStatus.Invalidated) - .SetProperty(x => x.InvalidatedAt, now), ct).ConfigureAwait(false); - if (transaction is not null) - await transaction.CommitAsync(ct).ConfigureAwait(false); - } - finally - { - if (transaction is not null) - await transaction.DisposeAsync().ConfigureAwait(false); - } - } - - private async Task ApplyLifecycleAsync(long installationId, GitHubWebhookPayload payload, CancellationToken ct) - { - var installation = await db.GitHubInstallations.FindAsync([installationId], ct).ConfigureAwait(false); - if (installation is null) - return; // A delivery can never create a project binding from untrusted display data. - - var now = DateTimeOffset.UtcNow; - if (payload.Action is "deleted" or "suspend") - { - installation.RevokedAt = now; - await db.GitHubRepositoryGrants.Where(x => x.InstallationId == installationId && x.RevokedAt == null) - .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); - return; - } - if (payload.Action is "created" or "unsuspend") - installation.RevokedAt = null; - - foreach (var repository in payload.RepositoriesRemoved ?? []) - { - if (repository.Id > 0) - await db.GitHubRepositoryGrants.Where(x => x.InstallationId == installationId && x.RepositoryId == repository.Id) - .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); - } - } -} +using System.Net.Http.Headers; +using System.Net.Http.Json; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Agentweaver.Api.Auth; +using Agentweaver.Api.Memory; +using Microsoft.EntityFrameworkCore; +using Microsoft.IdentityModel.JsonWebTokens; +using Microsoft.IdentityModel.Tokens; + +namespace Agentweaver.Api.Webhooks; + +public enum RepoAppInstallationOutcome { Success, InstallationUnavailable, ConfigurationUnavailable, ProviderUnavailable } +internal enum RepoAppInstallationBindingOutcome { Bound, PermissionChanged, Conflict } + +internal sealed record RepoAppInstallationAuthority( + long InstallationId, + long RepositoryId, + string FullNameDisplay, + IReadOnlyDictionary Permissions); +internal sealed record RepoAppInstallationToken(string Value, DateTimeOffset? ExpiresAt); + +/// +/// API-only boundary for a short-lived Repo App JWT and the single-repository installation +/// token it mints. Neither credential is written to persistence, logs, or HTTP responses. +/// +public sealed class RepoAppInstallationTokenService( + IConfiguration configuration, + MemoryDbContext db, + ISecretStore secretStore, + IHttpClientFactory httpClientFactory) +{ + private static readonly TimeSpan JwtLifetime = TimeSpan.FromMinutes(9); + private static readonly IReadOnlyDictionary UnattendedRepositoryPermissionCeilings = + new Dictionary(StringComparer.Ordinal) + { + ["contents"] = "write", + ["pull_requests"] = "write", + }; + private static readonly IReadOnlyDictionary RepositoryMetadataPermissionScope = + new Dictionary(StringComparer.Ordinal) + { + ["metadata"] = "read", + }; + + public async Task MintForRepositoryAsync( + long installationId, + long repositoryId, + Func useToken, + CancellationToken ct = default) + { + if (installationId <= 0 || repositoryId <= 0) + return RepoAppInstallationOutcome.InstallationUnavailable; + + var installationActive = await db.GitHubInstallations.AsNoTracking() + .AnyAsync(x => x.InstallationId == installationId && + x.AppKind == GitHubAppKind.Repo && + x.RevokedAt == null, ct).ConfigureAwait(false); + var grant = await db.GitHubRepositoryGrants.AsNoTracking() + .SingleOrDefaultAsync(x => x.InstallationId == installationId && + x.RepositoryId == repositoryId && + x.RevokedAt == null, ct).ConfigureAwait(false); + if (!installationActive || grant is null) + return RepoAppInstallationOutcome.InstallationUnavailable; + + var authority = await GetRepositoryAuthorityAsync(installationId, repositoryId, ct).ConfigureAwait(false); + if (authority is null) + return RepoAppInstallationOutcome.ProviderUnavailable; + if (!CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(grant.PermissionDigest), + Encoding.UTF8.GetBytes(CreatePermissionDigest(authority.Permissions)))) + { + await new RepoAppInstallationLifecycleService(db) + .InvalidateForPermissionChangeAsync(installationId, repositoryId, ct).ConfigureAwait(false); + return RepoAppInstallationOutcome.InstallationUnavailable; + } + if (!TryCreateUnattendedPermissionScope(authority.Permissions, out var requestedPermissions)) + return RepoAppInstallationOutcome.InstallationUnavailable; + + try + { + var appJwt = await CreateAppJwtAsync(ct).ConfigureAwait(false); + if (appJwt is null) + return RepoAppInstallationOutcome.ConfigurationUnavailable; + var installationToken = await GetInstallationTokenAsync( + appJwt, installationId, repositoryId, requestedPermissions, ct).ConfigureAwait(false); + if (installationToken is null) + return RepoAppInstallationOutcome.ProviderUnavailable; + + if (installationToken.ExpiresAt is null || installationToken.ExpiresAt <= DateTimeOffset.UtcNow) + return RepoAppInstallationOutcome.ProviderUnavailable; + await useToken(installationToken.Value, installationToken.ExpiresAt.Value).ConfigureAwait(false); + return RepoAppInstallationOutcome.Success; + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return RepoAppInstallationOutcome.ProviderUnavailable; + } + catch (HttpRequestException) + { + return RepoAppInstallationOutcome.ProviderUnavailable; + } + } + + public async Task VerifyRepositoryInstallationAsync( + long installationId, + long repositoryId, + CancellationToken ct = default) + => await GetRepositoryAuthorityAsync(installationId, repositoryId, ct).ConfigureAwait(false) is not null; + + /// Revokes a minted installation credential. This method does not persist or log it. + public async Task RevokeRepositoryTokenAsync(string token, CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(token)) + return; + + try + { + using var request = CreateGitHubRequest(HttpMethod.Delete, "/installation/token", token); + using var response = await httpClientFactory.CreateClient("github").SendAsync(request, ct) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (HttpRequestException) + { + // Token expiry is the backstop. Release and orphan cleanup must not fail on revoke. + } + } + + /// + /// Resolves the installation's exact repository authority from GitHub. The request supplies + /// only numeric identifiers; permissions and the display name are provider-owned values. + /// + internal async Task GetRepositoryAuthorityAsync( + long installationId, + long repositoryId, + CancellationToken ct = default) + { + if (installationId <= 0 || repositoryId <= 0) + return null; + + var appJwt = await CreateAppJwtAsync(ct).ConfigureAwait(false); + if (appJwt is null) + return null; + + try + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeout.CancelAfter(TimeSpan.FromSeconds(10)); + var client = httpClientFactory.CreateClient("github"); + using var installationRequest = CreateGitHubRequest( + HttpMethod.Get, $"/repositories/{repositoryId}/installation", appJwt); + using var installationResponse = await client.SendAsync(installationRequest, timeout.Token).ConfigureAwait(false); + if (!installationResponse.IsSuccessStatusCode) + return null; + using var installationDocument = JsonDocument.Parse( + await installationResponse.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); + var installation = installationDocument.RootElement; + if (!installation.TryGetProperty("id", out var actualInstallation) || + !actualInstallation.TryGetInt64(out var actualInstallationId) || + actualInstallationId != installationId || + !installation.TryGetProperty("repository_selection", out var repositorySelection) || + repositorySelection.ValueKind != JsonValueKind.String || + string.IsNullOrWhiteSpace(repositorySelection.GetString()) || + !installation.TryGetProperty("account", out var account) || + account.ValueKind != JsonValueKind.Object || + !TryGetNormalizedPermissions(installation, out var permissions)) + return null; + + var metadataToken = await GetInstallationTokenAsync( + appJwt, installationId, repositoryId, RepositoryMetadataPermissionScope, timeout.Token) + .ConfigureAwait(false); + if (metadataToken is null) + return null; + using var repositoryRequest = CreateGitHubRequest( + HttpMethod.Get, $"/repositories/{repositoryId}", metadataToken.Value); + using var repositoryResponse = await client.SendAsync(repositoryRequest, timeout.Token).ConfigureAwait(false); + if (!repositoryResponse.IsSuccessStatusCode) + return null; + using var repositoryDocument = JsonDocument.Parse( + await repositoryResponse.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); + var repository = repositoryDocument.RootElement; + if (!repository.TryGetProperty("id", out var actualRepository) || + !actualRepository.TryGetInt64(out var actualRepositoryId) || + actualRepositoryId != repositoryId || + !repository.TryGetProperty("full_name", out var fullName) || + fullName.ValueKind != JsonValueKind.String || + string.IsNullOrWhiteSpace(fullName.GetString())) + return null; + + return new RepoAppInstallationAuthority( + installationId, repositoryId, fullName.GetString()!, permissions); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return null; + } + catch (HttpRequestException) + { + return null; + } + catch (JsonException) + { + return null; + } + } + + private async Task CreateAppJwtAsync(CancellationToken ct) + { + if (!long.TryParse(configuration["Auth:RepoApp:AppId"], out var appId) || appId <= 0 || + string.IsNullOrWhiteSpace(configuration["Auth:RepoApp:PrivateKeySecretName"])) + return null; + var pem = await secretStore.GetSecretAsync(configuration["Auth:RepoApp:PrivateKeySecretName"]!, ct) + .ConfigureAwait(false); + if (!pem.Found || string.IsNullOrWhiteSpace(pem.Value)) + return null; + try + { + return CreateAppJwt(appId, pem.Value); + } + catch (CryptographicException) + { + return null; + } + } + + private HttpRequestMessage CreateGitHubRequest(HttpMethod method, string path, string appJwt) + { + var request = new HttpRequestMessage( + method, $"{(configuration["Auth:RepoApp:ApiUrl"] ?? "https://api.github.com").TrimEnd('/')}{path}"); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", appJwt); + request.Headers.UserAgent.ParseAdd("Agentweaver/1.0"); + request.Headers.Accept.ParseAdd("application/vnd.github+json"); + return request; + } + + private async Task GetInstallationTokenAsync( + string appJwt, + long installationId, + long repositoryId, + IReadOnlyDictionary permissions, + CancellationToken ct) + { + using var request = CreateGitHubRequest( + HttpMethod.Post, $"/app/installations/{installationId}/access_tokens", appJwt); + request.Content = JsonContent.Create(new { repository_ids = new[] { repositoryId }, permissions }); + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(ct); + timeout.CancelAfter(TimeSpan.FromSeconds(10)); + using var response = await httpClientFactory.CreateClient("github").SendAsync(request, timeout.Token) + .ConfigureAwait(false); + if (!response.IsSuccessStatusCode) + return null; + using var document = JsonDocument.Parse( + await response.Content.ReadAsStreamAsync(timeout.Token).ConfigureAwait(false)); + if (!document.RootElement.TryGetProperty("token", out var token) || + string.IsNullOrWhiteSpace(token.GetString())) + return null; + DateTimeOffset? expiresAt = document.RootElement.TryGetProperty("expires_at", out var expiresAtElement) && + DateTimeOffset.TryParse(expiresAtElement.GetString(), out var parsedExpiry) + ? parsedExpiry + : null; + return new(token.GetString()!, expiresAt); + } + + private static bool TryGetNormalizedPermissions( + JsonElement installation, + out IReadOnlyDictionary permissions) + { + permissions = new Dictionary(); + if (!installation.TryGetProperty("permissions", out var source) || + source.ValueKind != JsonValueKind.Object) + return false; + + var normalized = new Dictionary(StringComparer.Ordinal); + foreach (var permission in source.EnumerateObject()) + { + if (permission.Value.ValueKind != JsonValueKind.String) + return false; + var name = permission.Name.Trim().ToLowerInvariant(); + var value = permission.Value.GetString()?.Trim().ToLowerInvariant(); + if (string.IsNullOrWhiteSpace(name) || string.IsNullOrWhiteSpace(value) || + !normalized.TryAdd(name, value)) + return false; + } + permissions = normalized; + return normalized.Count > 0; + } + + private static bool TryCreateUnattendedPermissionScope( + IReadOnlyDictionary providerPermissions, + out IReadOnlyDictionary requestedPermissions) + { + var requested = new Dictionary(StringComparer.Ordinal); + foreach (var ceiling in UnattendedRepositoryPermissionCeilings) + { + if (!providerPermissions.TryGetValue(ceiling.Key, out var actual)) + continue; + if (!string.Equals(actual, "read", StringComparison.Ordinal) && + !string.Equals(actual, "write", StringComparison.Ordinal)) + { + requestedPermissions = new Dictionary(); + return false; + } + if (string.Equals(ceiling.Value, "read", StringComparison.Ordinal) && + string.Equals(actual, "write", StringComparison.Ordinal)) + { + requestedPermissions = new Dictionary(); + return false; + } + requested[ceiling.Key] = actual; + } + requestedPermissions = requested; + return requested.Count > 0; + } + + internal static string CreateAppJwt(long appId, string pem) + { + using var rsa = RSA.Create(); + rsa.ImportFromPem(pem); + var now = DateTime.UtcNow; + var signingKey = new RsaSecurityKey(rsa) + { + CryptoProviderFactory = new CryptoProviderFactory { CacheSignatureProviders = false }, + }; + return new JsonWebTokenHandler().CreateToken(new SecurityTokenDescriptor + { + Issuer = appId.ToString(System.Globalization.CultureInfo.InvariantCulture), + IssuedAt = now.AddMinutes(-1), + NotBefore = now.AddMinutes(-1), + Expires = now.Add(JwtLifetime), + SigningCredentials = new SigningCredentials(signingKey, SecurityAlgorithms.RsaSha256), + }); + } + + internal static string CreatePermissionDigest(IReadOnlyDictionary permissions) + { + var canonical = string.Join("&", permissions.OrderBy(x => x.Key, StringComparer.Ordinal) + .Select(x => $"{x.Key.Trim().ToLowerInvariant()}={x.Value.Trim().ToLowerInvariant()}")); + return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(canonical))).ToLowerInvariant(); + } +} + +/// Durable installation/grant state machine for authenticated Repo App deliveries. +public sealed class RepoAppInstallationLifecycleService(MemoryDbContext db) +{ + private const string CompletedEventPrefix = "completed/"; + private static readonly TimeSpan ProcessingLease = TimeSpan.FromMinutes(10); + + public async Task<(bool Claimed, IReadOnlyList ProjectIds)> ProcessAsync( + string deliveryId, + string eventName, + GitHubWebhookPayload payload, + CancellationToken ct = default) + { + if (string.IsNullOrWhiteSpace(deliveryId)) + return (false, []); + + db.ChangeTracker.Clear(); + await using var transaction = await db.Database.BeginTransactionAsync(ct).ConfigureAwait(false); + db.GitHubLifecycleDeliveries.Add(new GitHubLifecycleDeliveryRecord + { + DeliveryId = deliveryId, + EventName = eventName, + InstallationId = payload.Installation?.Id, + RepositoryId = payload.Repository?.Id, + ReceivedAt = DateTimeOffset.UtcNow, + }); + try + { + await db.SaveChangesAsync(ct).ConfigureAwait(false); + } + catch (DbUpdateException) + { + await transaction.RollbackAsync(CancellationToken.None).ConfigureAwait(false); + db.ChangeTracker.Clear(); + var leaseExpiresBefore = DateTimeOffset.UtcNow.Subtract(ProcessingLease); + var abandoned = await db.GitHubLifecycleDeliveries.FindAsync([deliveryId], ct).ConfigureAwait(false); + if (abandoned is null || abandoned.EventName != eventName || abandoned.ReceivedAt >= leaseExpiresBefore) + return (false, []); + var reclaimed = await db.GitHubLifecycleDeliveries + .Where(x => x.DeliveryId == deliveryId && + x.EventName == abandoned.EventName && + x.ReceivedAt == abandoned.ReceivedAt) + .ExecuteDeleteAsync(ct).ConfigureAwait(false); + if (reclaimed != 1) + return (false, []); + return await ProcessAsync(deliveryId, eventName, payload, ct).ConfigureAwait(false); + } + + var installationId = (payload.Installation?.Id).GetValueOrDefault(); + if (installationId > 0 && eventName is "installation" or "installation_repositories") + { + await ApplyLifecycleAsync(installationId, payload, ct).ConfigureAwait(false); + await db.SaveChangesAsync(ct).ConfigureAwait(false); + } + + db.ChangeTracker.Clear(); + var installationActive = installationId > 0 && await db.GitHubInstallations.AsNoTracking() + .AnyAsync(x => x.InstallationId == installationId && + x.AppKind == GitHubAppKind.Repo && + x.RevokedAt == null, ct).ConfigureAwait(false); + var projectIds = installationActive && payload.Repository?.Id is > 0 + ? await db.GitHubRepositoryGrants.AsNoTracking() + .Where(x => x.InstallationId == installationId && + x.RepositoryId == payload.Repository.Id && + x.RevokedAt == null) + .Select(x => x.ProjectId).ToListAsync(ct).ConfigureAwait(false) + : []; + await db.SaveChangesAsync(ct).ConfigureAwait(false); + await transaction.CommitAsync(ct).ConfigureAwait(false); + return (true, projectIds); + } + + /// + /// Releases a claim only when downstream dispatch did not complete, allowing GitHub to retry. + /// The dispatch path has its own delivery-id idempotency guard. + /// + public async Task ReleaseAsync(string deliveryId, CancellationToken ct = default) + { + db.ChangeTracker.Clear(); + await db.GitHubLifecycleDeliveries.Where(x => x.DeliveryId == deliveryId) + .ExecuteDeleteAsync(ct).ConfigureAwait(false); + } + + public Task IsCompletedAsync(string deliveryId, CancellationToken ct = default) => + db.GitHubLifecycleDeliveries.AsNoTracking() + .AnyAsync(x => x.DeliveryId == deliveryId && + x.EventName.StartsWith(CompletedEventPrefix), ct); + + public async Task CompleteAsync(string deliveryId, CancellationToken ct = default) + { + db.ChangeTracker.Clear(); + var current = await db.GitHubLifecycleDeliveries.AsNoTracking() + .Where(x => x.DeliveryId == deliveryId) + .Select(x => x.EventName).SingleOrDefaultAsync(ct).ConfigureAwait(false); + if (current is null) + return false; + if (current.StartsWith(CompletedEventPrefix, StringComparison.Ordinal)) + return true; + return await db.GitHubLifecycleDeliveries.Where(x => x.DeliveryId == deliveryId && x.EventName == current) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.EventName, $"{CompletedEventPrefix}{current}"), ct) + .ConfigureAwait(false) == 1; + } + + internal async Task BindAsync( + string projectId, + RepoAppInstallationAuthority authority, + CancellationToken ct = default) + { + await using var transaction = await db.Database.BeginTransactionAsync( + System.Data.IsolationLevel.Serializable, ct).ConfigureAwait(false); + var now = DateTimeOffset.UtcNow; + var installation = await db.GitHubInstallations.FindAsync([authority.InstallationId], ct).ConfigureAwait(false); + if (installation is not null && installation.ProjectId is not null && + !string.Equals(installation.ProjectId, projectId, StringComparison.Ordinal)) + return RepoAppInstallationBindingOutcome.Conflict; + if (installation is null) + db.GitHubInstallations.Add(new GitHubInstallationRecord + { + InstallationId = authority.InstallationId, AppKind = GitHubAppKind.Repo, ProjectId = projectId, CreatedAt = now, + }); + else + { + installation.ProjectId = projectId; + installation.RevokedAt = null; + } + + var grant = await db.GitHubRepositoryGrants.FindAsync( + [authority.InstallationId, authority.RepositoryId], ct).ConfigureAwait(false); + if (grant is not null && !string.Equals(grant.ProjectId, projectId, StringComparison.Ordinal)) + return RepoAppInstallationBindingOutcome.Conflict; + if (grant is null) + db.GitHubRepositoryGrants.Add(new GitHubRepositoryGrantRecord + { + InstallationId = authority.InstallationId, RepositoryId = authority.RepositoryId, ProjectId = projectId, + FullNameDisplay = authority.FullNameDisplay, + PermissionDigest = RepoAppInstallationTokenService.CreatePermissionDigest(authority.Permissions), + GrantedAt = now, + }); + else + { + var permissionDigest = RepoAppInstallationTokenService.CreatePermissionDigest(authority.Permissions); + if (!CryptographicOperations.FixedTimeEquals( + Encoding.UTF8.GetBytes(grant.PermissionDigest), Encoding.UTF8.GetBytes(permissionDigest))) + { + grant.FullNameDisplay = authority.FullNameDisplay; + grant.RevokedAt = now; + await InvalidateForPermissionChangeAsync(authority.InstallationId, authority.RepositoryId, ct) + .ConfigureAwait(false); + await db.SaveChangesAsync(ct).ConfigureAwait(false); + await transaction.CommitAsync(ct).ConfigureAwait(false); + return RepoAppInstallationBindingOutcome.PermissionChanged; + } + grant.FullNameDisplay = authority.FullNameDisplay; + grant.RevokedAt = null; + } + try + { + await db.SaveChangesAsync(ct).ConfigureAwait(false); + await transaction.CommitAsync(ct).ConfigureAwait(false); + return RepoAppInstallationBindingOutcome.Bound; + } + catch (DbUpdateException) + { + db.ChangeTracker.Clear(); + return RepoAppInstallationBindingOutcome.Conflict; + } + } + + public async Task InvalidateForPermissionChangeAsync( + long installationId, + long repositoryId, + CancellationToken ct = default) + { + var transaction = db.Database.CurrentTransaction is null + ? await db.Database.BeginTransactionAsync(ct).ConfigureAwait(false) + : null; + var now = DateTimeOffset.UtcNow; + try + { + await db.GitHubRepositoryGrants + .Where(x => x.InstallationId == installationId && + x.RepositoryId == repositoryId && + x.RevokedAt == null) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); + await db.AutomationActivations + .Where(x => x.InstallationId == installationId && + x.RepositoryId == repositoryId && + x.Status != AutomationActivationStatus.Invalidated) + .ExecuteUpdateAsync(s => s + .SetProperty(x => x.Status, AutomationActivationStatus.Invalidated) + .SetProperty(x => x.InvalidatedAt, now), ct).ConfigureAwait(false); + if (transaction is not null) + await transaction.CommitAsync(ct).ConfigureAwait(false); + } + finally + { + if (transaction is not null) + await transaction.DisposeAsync().ConfigureAwait(false); + } + } + + private async Task ApplyLifecycleAsync(long installationId, GitHubWebhookPayload payload, CancellationToken ct) + { + var installation = await db.GitHubInstallations.FindAsync([installationId], ct).ConfigureAwait(false); + if (installation is null) + return; // A delivery can never create a project binding from untrusted display data. + + var now = DateTimeOffset.UtcNow; + if (payload.Action is "deleted" or "suspend") + { + installation.RevokedAt = now; + await db.GitHubRepositoryGrants.Where(x => x.InstallationId == installationId && x.RevokedAt == null) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); + return; + } + if (payload.Action is "created" or "unsuspend") + installation.RevokedAt = null; + + foreach (var repository in payload.RepositoriesRemoved ?? []) + { + if (repository.Id > 0) + await db.GitHubRepositoryGrants.Where(x => x.InstallationId == installationId && x.RepositoryId == repository.Id) + .ExecuteUpdateAsync(s => s.SetProperty(x => x.RevokedAt, now), ct).ConfigureAwait(false); + } + } +} From 4c9b28a49e5be4dd42363f76d6d95b506edd917f Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 20:48:57 -0700 Subject: [PATCH 03/12] fix: harden sandbox repository credentials Launch credential-bearing git and gh commands directly, retain failed revocations for retry, and require approval for sensitive gh credential commands. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- ...sandbox-repository-credential-hardening.md | 7 + .../Infrastructure/YamlSandboxPolicyStore.cs | 2 + .../Sandbox/KubernetesSandboxExecutor.cs | 31 ++- .../RunRepositoryCredentialRegistry.cs | 102 +++++--- .../Webhooks/RepoAppInstallationService.cs | 18 +- docs/deep-dive/sandboxed-execution.md | 17 +- .../Tools/RunCommandTool.cs | 230 ++++++++++++++++-- packages/Agentweaver.Domain/SandboxPolicy.cs | 2 +- .../ISandboxExecutor.cs | 17 +- .../KataBwrapExecutor.cs | 19 +- .../LinuxBwrapExecutor.cs | 116 +++++++++ .../LinuxNativeMxcSandboxExecutor.cs | 10 + .../MxcSandboxExecutor.cs | 10 + .../PassthroughExecutor.cs | 10 +- .../PodExec/PodExecProtocol.cs | 3 + .../PodExec/PodExecSandboxClient.cs | 6 + .../PodExec/PodExecServer.cs | 8 +- .../SandboxCommandEnvironment.cs | 16 ++ .../WslMxcSandboxExecutor.cs | 116 +++++++++ .../Auth/TwoAppCredentialArchitectureTests.cs | 2 +- .../AssemblyBuildTestShellGuardTests.cs | 151 +++++++++++- .../RunRepositoryCredentialRegistryTests.cs | 67 +++++ .../RepoAppInstallationServiceTests.cs | 46 +++- 23 files changed, 909 insertions(+), 97 deletions(-) create mode 100644 .changeset/sandbox-repository-credential-hardening.md create mode 100644 tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs diff --git a/.changeset/sandbox-repository-credential-hardening.md b/.changeset/sandbox-repository-credential-hardening.md new file mode 100644 index 000000000..c5800e63b --- /dev/null +++ b/.changeset/sandbox-repository-credential-hardening.md @@ -0,0 +1,7 @@ +--- + +"agentweaver": patch + +--- + +Harden sandbox repository credential delivery by starting validated GitHub CLI commands directly and retrying failed credential revocation during run cleanup. diff --git a/apps/Agentweaver.Api/Infrastructure/YamlSandboxPolicyStore.cs b/apps/Agentweaver.Api/Infrastructure/YamlSandboxPolicyStore.cs index f4600c755..90fab2f1e 100644 --- a/apps/Agentweaver.Api/Infrastructure/YamlSandboxPolicyStore.cs +++ b/apps/Agentweaver.Api/Infrastructure/YamlSandboxPolicyStore.cs @@ -165,6 +165,8 @@ internal sealed class SandboxPolicyYamlDto "git push origin --delete", "git push --delete", "git branch -D", "git clean -fd", "git clean -fxd", + // GitHub credential commands + "gh secret set", "gh auth token", // PowerShell destructive "Remove-Item -Recurse", "Remove-Item -Force", "ri -r", "ri -Recurse", "Format-Volume", "Clear-Disk", diff --git a/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs b/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs index e858b33bb..4165f795c 100644 --- a/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs +++ b/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs @@ -1389,11 +1389,11 @@ private async Task ExecInPodAsync( { const int maxOutputBytes = 4 * 1024 * 1024; - var shellScript = BuildShellScript(command, podWorkingDirectory); + var execCommand = BuildExecCommand(command, podWorkingDirectory); var ws = await _client.WebSocketNamespacedPodExecAsync( podName, _options.Namespace, - new[] { "/bin/sh", "-c", shellScript }, + execCommand, container: ContainerName, stdin: false, stdout: true, stderr: true, tty: false, cancellationToken: ct); @@ -1534,6 +1534,33 @@ private static string BuildShellScript(SandboxCommand command, string podWorking return sb.ToString(); } + private static string[] BuildExecCommand(SandboxCommand command, string podWorkingDirectory) + { + if (command.DirectExecution is not { } directExecution) + return ["/bin/sh", "-c", BuildShellScript(command, podWorkingDirectory)]; + + var environment = new Dictionary(StringComparer.Ordinal); + if (command.Environment is { Count: > 0 }) + { + foreach (var (key, value) in command.Environment) + environment[key] = value; + } + if (directExecution.Environment is { Count: > 0 }) + { + foreach (var (key, value) in directExecution.Environment) + environment[key] = value; + } + + if (environment.Count == 0) + return [directExecution.Executable, .. directExecution.Arguments]; + + var execCommand = new List { "/usr/bin/env", "-i", "PATH=/usr/local/bin:/usr/bin:/bin" }; + execCommand.AddRange(environment.Select(pair => $"{pair.Key}={pair.Value}")); + execCommand.Add(directExecution.Executable); + execCommand.AddRange(directExecution.Arguments); + return [.. execCommand]; + } + private static string ShellSingleQuote(string s) => "'" + s.Replace("'", "'\\''") + "'"; } diff --git a/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs index 1b86bbdcf..ef4d3ef21 100644 --- a/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs +++ b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs @@ -11,11 +11,20 @@ namespace Agentweaver.Api.Sandbox; /// Holds minted repository credentials in API memory until the owning run releases its pod. /// The registry has no command inputs and does not persist credentials. /// -public sealed class RunRepositoryCredentialRegistry(IServiceScopeFactory scopeFactory) +public sealed class RunRepositoryCredentialRegistry { - private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); + private readonly IRunRepositoryCredentialMinter _credentialMinter; + private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _mintLocks = new(StringComparer.Ordinal); + public RunRepositoryCredentialRegistry(IServiceScopeFactory scopeFactory) + : this(new RunRepositoryCredentialMinter(scopeFactory)) + { + } + + internal RunRepositoryCredentialRegistry(IRunRepositoryCredentialMinter credentialMinter) => + _credentialMinter = credentialMinter; + public async Task MintAsync(string runId, CancellationToken ct = default) { var mintLock = _mintLocks.GetOrAdd(runId, static _ => new SemaphoreSlim(1, 1)); @@ -29,25 +38,8 @@ public sealed class RunRepositoryCredentialRegistry(IServiceScopeFactory scopeFa _entries.TryRemove(runId, out _); } - using var scope = scopeFactory.CreateScope(); - var persistence = scope.ServiceProvider.GetRequiredService(); - var snapshot = (await persistence.GetCapabilitySnapshotsAsync(runId, ct).ConfigureAwait(false)) - .SingleOrDefault(x => x.Purpose == GitHubCapabilityPurpose.UnattendedRepository); - if (snapshot is null) - return null; - - Entry? minted = null; - var outcome = await scope.ServiceProvider.GetRequiredService() - .TryUseRepositoryCredentialAsync( - new SnapshotRef(snapshot.SnapshotRef), - DateTimeOffset.UtcNow, - (token, expiresAt) => - { - minted = new Entry(token, expiresAt); - return Task.CompletedTask; - }, - ct).ConfigureAwait(false); - if (outcome != GitHubCapabilityBrokerOutcome.Issued || minted is null) + var minted = await _credentialMinter.MintAsync(runId, ct).ConfigureAwait(false); + if (minted is null || minted.ExpiresAt <= DateTimeOffset.UtcNow) return null; _entries[runId] = minted; @@ -68,26 +60,68 @@ public async Task RevokeAsync(string? runId, CancellationToken ct = default) await mintLock.WaitAsync(ct).ConfigureAwait(false); try { - if (!_entries.TryRemove(runId, out var entry)) + if (!_entries.TryGetValue(runId, out var entry)) return; - using var scope = scopeFactory.CreateScope(); - await scope.ServiceProvider.GetRequiredService() - .RevokeRepositoryTokenAsync(entry.AccessToken, ct).ConfigureAwait(false); - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) - { - throw; - } - catch - { - // Token expiry bounds a failed best-effort revoke. + if (entry.ExpiresAt <= DateTimeOffset.UtcNow) + { + _entries.TryRemove(runId, out _); + return; + } + + await _credentialMinter.RevokeAsync(entry.AccessToken, ct).ConfigureAwait(false); + _entries.TryRemove(runId, out _); } finally { mintLock.Release(); } } +} + +/// +/// The registry's credential-only dependency. It has no knowledge of git, gh, command text, or +/// sandbox execution; it only mints from the run's fenced repository snapshot and revokes the +/// provider credential. +/// +internal interface IRunRepositoryCredentialMinter +{ + Task MintAsync(string runId, CancellationToken ct); + Task RevokeAsync(string accessToken, CancellationToken ct); +} + +internal sealed class RunRepositoryCredentialMinter(IServiceScopeFactory scopeFactory) + : IRunRepositoryCredentialMinter +{ + public async Task MintAsync(string runId, CancellationToken ct) + { + using var scope = scopeFactory.CreateScope(); + var persistence = scope.ServiceProvider.GetRequiredService(); + var snapshot = (await persistence.GetCapabilitySnapshotsAsync(runId, ct).ConfigureAwait(false)) + .SingleOrDefault(x => x.Purpose == GitHubCapabilityPurpose.UnattendedRepository); + if (snapshot is null) + return null; + + RepositoryCredential? minted = null; + var outcome = await scope.ServiceProvider.GetRequiredService() + .TryUseRepositoryCredentialAsync( + new SnapshotRef(snapshot.SnapshotRef), + DateTimeOffset.UtcNow, + (token, expiresAt) => + { + minted = new RepositoryCredential(token, expiresAt); + return Task.CompletedTask; + }, + ct).ConfigureAwait(false); + return outcome == GitHubCapabilityBrokerOutcome.Issued ? minted : null; + } - private sealed record Entry(string AccessToken, DateTimeOffset ExpiresAt); + public async Task RevokeAsync(string accessToken, CancellationToken ct) + { + using var scope = scopeFactory.CreateScope(); + await scope.ServiceProvider.GetRequiredService() + .RevokeRepositoryTokenAsync(accessToken, ct).ConfigureAwait(false); + } } + +internal sealed record RepositoryCredential(string AccessToken, DateTimeOffset ExpiresAt); diff --git a/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs b/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs index ca32f8e90..99e60cc87 100644 --- a/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs +++ b/apps/Agentweaver.Api/Webhooks/RepoAppInstallationService.cs @@ -115,20 +115,10 @@ public async Task RevokeRepositoryTokenAsync(string token, CancellationToken ct if (string.IsNullOrWhiteSpace(token)) return; - try - { - using var request = CreateGitHubRequest(HttpMethod.Delete, "/installation/token", token); - using var response = await httpClientFactory.CreateClient("github").SendAsync(request, ct) - .ConfigureAwait(false); - } - catch (OperationCanceledException) when (ct.IsCancellationRequested) - { - throw; - } - catch (HttpRequestException) - { - // Token expiry is the backstop. Release and orphan cleanup must not fail on revoke. - } + using var request = CreateGitHubRequest(HttpMethod.Delete, "/installation/token", token); + using var response = await httpClientFactory.CreateClient("github").SendAsync(request, ct) + .ConfigureAwait(false); + response.EnsureSuccessStatusCode(); } /// diff --git a/docs/deep-dive/sandboxed-execution.md b/docs/deep-dive/sandboxed-execution.md index 7e045e2f0..5e5cee92d 100644 --- a/docs/deep-dive/sandboxed-execution.md +++ b/docs/deep-dive/sandboxed-execution.md @@ -84,11 +84,18 @@ The API (`GET /api/sandbox-policy`, `PUT /api/sandbox-policy`) reads and writes The policy is read through `ISandboxPolicyStore.GetPolicyAsync` and is configurable via the API at `GET /api/sandbox-policy` and `PUT /api/sandbox-policy`. See [sandbox-setup.md](../reference/sandbox-setup.md) for operator instructions. -The API sends one short-lived installation credential for the selected repository and run. The sandbox gives it only to one `git` or `gh` command. - -The approval policy gates `git push`, remote changes, `gh pr` changes, `gh repo` changes, `gh api`, and `gh auth` commands. The API does not inspect or proxy these commands. - -The system keeps this credential out of pod specs, files, logs, events, annotations, shared environments, and credential-helper files. Normal release and orphan cleanup revoke it on a best-effort basis. Token expiry limits a failed revoke. +The API sends one short-lived installation credential for the selected repository and run. When +that credential is used, the sandbox parses the command and starts the approved `git` or `gh` +executable directly instead of placing the credential in a shell environment. Shell syntax, +Git aliases, configured helpers, alternate worktrees, and helper-selecting Git options are +rejected for this path. Git runs with hooks and credential helpers disabled; its GitHub +authorization header is scoped to the directly-started Git process. + +The approval policy gates `git push`, remote changes, `gh pr` changes, `gh repo` changes, `gh api`, +`gh secret set`, and `gh auth` commands (including `gh auth token`). The API does not inspect or +proxy these commands. + +The system keeps this credential out of pod specs, files, logs, events, annotations, shared environments, and credential-helper files. Normal release and orphan cleanup log failed revocations and retain the in-memory retry state until a later cleanup succeeds or the credential actually expires. ## Security model diff --git a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs index 783c9cee9..0c65f5869 100644 --- a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs +++ b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs @@ -99,9 +99,13 @@ public AIFunction CreateFunction(SandboxToolContext ctx) => timeout = ctx.Options.MinimumTimeoutMs; if (ctx.Options.MaximumTimeoutMs > 0) timeout = Math.Min(timeout, ctx.Options.MaximumTimeoutMs); - var environment = BuildCommandEnvironment(ctx.WorkingDirectory, scratchDirectory); - if (!TryAddRepositoryCredential(command, ctx.Options.RepositoryAccessToken, environment, out var credentialError)) + if (!TryCreateRepositoryCredentialCommand( + command, + ctx.Options.RepositoryAccessToken, + out var directExecution, + out var credentialError)) return credentialError!; + var environment = BuildCommandEnvironment(ctx.WorkingDirectory, scratchDirectory); var cmd = new SandboxCommand( command, @@ -110,7 +114,8 @@ public AIFunction CreateFunction(SandboxToolContext ctx) => fsPolicy, timeout, NetworkEnabled: ctx.Options.NetworkEnabled, - AgentweaverRunId: string.IsNullOrEmpty(ctx.RunId) ? null : ctx.RunId); + AgentweaverRunId: string.IsNullOrEmpty(ctx.RunId) ? null : ctx.RunId, + DirectExecution: directExecution); IDisposable? executionLease = null; SandboxExecResult result; @@ -191,46 +196,223 @@ private static Dictionary BuildCommandEnvironment( return environment; } - private static bool TryAddRepositoryCredential( + private static readonly HashSet BuiltInGitCommands = new(StringComparer.Ordinal) + { + "add", "apply", "blame", "branch", "checkout", "clean", "clone", "commit", + "diff", "fetch", "grep", "log", "ls-files", "ls-remote", "ls-tree", "merge", + "merge-base", "mv", "pull", "push", "rebase", "remote", "reset", "restore", + "revert", "rm", "show", "show-ref", "sparse-checkout", "stash", "status", + "switch", "tag", "worktree", + }; + + private static readonly HashSet BuiltInGhCommands = new(StringComparer.Ordinal) + { + "api", "attestation", "auth", "cache", "codespace", "completion", "config", + "gist", "gpg-key", "issue", "label", "org", "pr", "project", "release", "repo", + "ruleset", "search", "secret", "ssh-key", "status", "variable", "workflow", + }; + + private static bool TryCreateRepositoryCredentialCommand( string command, string? accessToken, - IDictionary environment, + out SandboxDirectExecution? directExecution, out string? error) { + directExecution = null; error = null; if (string.IsNullOrWhiteSpace(accessToken)) return true; - var trimmed = command.Trim(); - var isGitHubCommand = trimmed.Equals("git", StringComparison.Ordinal) || - trimmed.Equals("gh", StringComparison.Ordinal) || - trimmed.StartsWith("git ", StringComparison.Ordinal) || - trimmed.StartsWith("gh ", StringComparison.Ordinal); - if (!isGitHubCommand) + if (!TryParseCommand(command, out var arguments, out error)) + return false; + if (arguments.Count == 0 || + (arguments[0] != "git" && arguments[0] != "gh")) return true; - if (trimmed.IndexOfAny([';', '|', '&', '\r', '\n', '`', '<', '>']) >= 0 || - trimmed.Contains("$(", StringComparison.Ordinal)) + if (arguments[0] == "git") { - error = "Command rejected: GitHub credentials require one git or gh command."; - return false; + if (!TryValidateGitArguments(arguments, out error)) + return false; + + var basicAuthorization = Convert.ToBase64String( + System.Text.Encoding.UTF8.GetBytes($"x-access-token:{accessToken}")); + var gitArguments = new List + { + "--no-pager", + "-c", "credential.helper=", + "-c", "core.hooksPath=/dev/null", + "-c", "protocol.allow=never", + "-c", "protocol.https.allow=always", + "-c", $"http.https://github.com/.extraheader=AUTHORIZATION: basic {basicAuthorization}", + }; + gitArguments.AddRange(arguments.Skip(1)); + directExecution = new SandboxDirectExecution("git", gitArguments); + return true; } - // GH_TOKEN supports gh and `gh auth git-credential` supports HTTPS Git without a token file. - environment["GH_TOKEN"] = accessToken; - environment["GITHUB_TOKEN"] = accessToken; - environment["GIT_CONFIG_COUNT"] = "1"; - environment["GIT_CONFIG_KEY_0"] = "credential.helper"; - environment["GIT_CONFIG_VALUE_0"] = "!gh auth git-credential"; + if (!TryValidateGhArguments(arguments, out error)) + return false; + + directExecution = new SandboxDirectExecution( + "gh", + arguments.Skip(1).ToArray(), + new Dictionary(StringComparer.Ordinal) + { + ["GH_TOKEN"] = accessToken, + ["GH_PROMPT_DISABLED"] = "1", + }); return true; } private static string RedactOutput(string value, SandboxToolContext ctx) { var redacted = ctx.Redactor.Redact(value); - return string.IsNullOrWhiteSpace(ctx.Options.RepositoryAccessToken) - ? redacted - : redacted.Replace(ctx.Options.RepositoryAccessToken, "***", StringComparison.Ordinal); + if (string.IsNullOrWhiteSpace(ctx.Options.RepositoryAccessToken)) + return redacted; + + var basicAuthorization = Convert.ToBase64String( + System.Text.Encoding.UTF8.GetBytes($"x-access-token:{ctx.Options.RepositoryAccessToken}")); + return redacted + .Replace(ctx.Options.RepositoryAccessToken, "***", StringComparison.Ordinal) + .Replace(basicAuthorization, "***", StringComparison.Ordinal); + } + + private static bool TryValidateGitArguments( + IReadOnlyList arguments, + out string? error) + { + error = null; + if (arguments.Count < 2 || !BuiltInGitCommands.Contains(arguments[1])) + { + error = "Command rejected: repository credentials require a built-in git command."; + return false; + } + + foreach (var argument in arguments.Skip(2)) + { + if (!IsUnsafeGitArgument(argument)) + continue; + + error = "Command rejected: git configuration, aliases, helpers, and alternate worktrees are not allowed with repository credentials."; + return false; + } + + return true; + } + + private static bool IsUnsafeGitArgument(string argument) => + argument == "-c" || + argument.StartsWith("-c", StringComparison.Ordinal) || + argument == "-C" || + argument.StartsWith("--config", StringComparison.Ordinal) || + argument.StartsWith("--exec-path", StringComparison.Ordinal) || + argument.StartsWith("--git-dir", StringComparison.Ordinal) || + argument.StartsWith("--work-tree", StringComparison.Ordinal) || + argument.StartsWith("--namespace", StringComparison.Ordinal) || + argument.StartsWith("--upload-pack", StringComparison.Ordinal) || + argument.StartsWith("--receive-pack", StringComparison.Ordinal) || + argument.StartsWith("--git-upload-pack", StringComparison.Ordinal) || + argument.StartsWith("--git-receive-pack", StringComparison.Ordinal) || + argument.StartsWith("--paginate", StringComparison.Ordinal) || + argument == "-p" || + argument.StartsWith("ext::", StringComparison.OrdinalIgnoreCase); + + private static bool TryValidateGhArguments( + IReadOnlyList arguments, + out string? error) + { + error = null; + if (arguments.Count < 2 || !BuiltInGhCommands.Contains(arguments[1])) + { + error = "Command rejected: repository credentials require a built-in gh command."; + return false; + } + + var hasNestedCommand = (string topLevel, string nested) => + arguments[1] == topLevel && + arguments.Skip(2).Any(argument => argument == nested); + if (arguments[1] == "codespace" || + arguments.Skip(2).Any(argument => + argument is "--web" or "--browser" or "--editor") || + hasNestedCommand("auth", "setup-git") || + hasNestedCommand("repo", "clone") || + hasNestedCommand("pr", "checkout")) + { + error = "Command rejected: gh commands that start another executable are not allowed with repository credentials."; + return false; + } + + return true; + } + + private static bool TryParseCommand( + string command, + out IReadOnlyList arguments, + out string? error) + { + arguments = []; + error = null; + var parsed = new List(); + var current = new System.Text.StringBuilder(); + var quote = '\0'; + var escaping = false; + + foreach (var character in command) + { + if (character is '\r' or '\n' or '\0' or ';' or '|' or '&' or '`' or '$' or + '<' or '>' or '(' or ')' or '{' or '}' or '[' or ']' or '*' or '?' or '!' or '~') + { + error = "Command rejected: GitHub credentials require one direct git or gh command without shell metacharacters."; + return false; + } + + if (escaping) + { + current.Append(character); + escaping = false; + continue; + } + + if (character == '\\' && quote != '\'') + { + escaping = true; + continue; + } + + if (character is '\'' or '"') + { + if (quote == '\0') + quote = character; + else if (quote == character) + quote = '\0'; + else + current.Append(character); + continue; + } + + if (char.IsWhiteSpace(character) && quote == '\0') + { + if (current.Length > 0) + { + parsed.Add(current.ToString()); + current.Clear(); + } + continue; + } + + current.Append(character); + } + + if (escaping || quote != '\0') + { + error = "Command rejected: GitHub credentials require balanced, literal arguments."; + return false; + } + if (current.Length > 0) + parsed.Add(current.ToString()); + + arguments = parsed; + return true; } private static bool IsDestructivePattern(string command, string[] patterns) diff --git a/packages/Agentweaver.Domain/SandboxPolicy.cs b/packages/Agentweaver.Domain/SandboxPolicy.cs index 423dee792..64354761b 100644 --- a/packages/Agentweaver.Domain/SandboxPolicy.cs +++ b/packages/Agentweaver.Domain/SandboxPolicy.cs @@ -69,7 +69,7 @@ public sealed record SandboxPolicy // GitHub repository changes and credential commands "gh pr create", "gh pr merge", "gh pr close", "gh repo delete", "gh repo archive", - "gh api", "gh auth login", "gh auth logout", + "gh api", "gh secret set", "gh auth login", "gh auth logout", "gh auth token", // PowerShell destructive "Remove-Item -Recurse", "Remove-Item -Force", "ri -r", "ri -Recurse", "Format-Volume", "Clear-Disk", diff --git a/packages/Agentweaver.SandboxExec/ISandboxExecutor.cs b/packages/Agentweaver.SandboxExec/ISandboxExecutor.cs index f861988e4..c9dc58a62 100644 --- a/packages/Agentweaver.SandboxExec/ISandboxExecutor.cs +++ b/packages/Agentweaver.SandboxExec/ISandboxExecutor.cs @@ -46,7 +46,22 @@ public sealed record SandboxCommand( /// so that the preview /// port-forward service can locate the pod later. /// - string? AgentweaverRunId = null); + string? AgentweaverRunId = null, + /// + /// A parsed command that must be started directly rather than through a shell. This is used + /// only when a repository credential is present, keeping that credential out of shell + /// environments. + /// + SandboxDirectExecution? DirectExecution = null); + +/// +/// A command whose executable and arguments have already been validated by the sandbox tool. +/// is applied only to its directly-started executable. +/// +public sealed record SandboxDirectExecution( + string Executable, + IReadOnlyList Arguments, + IReadOnlyDictionary? Environment = null); /// /// Filesystem policy handed to the sandbox engine. DeniedPaths maps to diff --git a/packages/Agentweaver.SandboxExec/KataBwrapExecutor.cs b/packages/Agentweaver.SandboxExec/KataBwrapExecutor.cs index 20cd70479..1f45c2ddc 100644 --- a/packages/Agentweaver.SandboxExec/KataBwrapExecutor.cs +++ b/packages/Agentweaver.SandboxExec/KataBwrapExecutor.cs @@ -578,7 +578,13 @@ internal ProcessStartInfo BuildProcessStartInfo(SandboxCommand command) throw new PlatformNotSupportedException("Kata bubblewrap isolation requires Linux."); var mounts = BuildMountPlan(command); - var environment = BuildChildEnvironment(command); + var environment = BuildChildEnvironment(command) + .ToDictionary(pair => pair.Key, pair => pair.Value, StringComparer.Ordinal); + if (command.DirectExecution?.Environment is { Count: > 0 } directEnvironment) + { + foreach (var (key, value) in directEnvironment) + environment[key] = value; + } var runRootPid = EnsureWritableSystemRoot(command); var psi = new ProcessStartInfo { @@ -653,7 +659,16 @@ internal ProcessStartInfo BuildProcessStartInfo(SandboxCommand command) // group, so the workload must be exec'd directly: an extra /usr/bin/setsid would fork a // grandchild whose pid bwrap never reports, and the Kata guest kernel has no // /proc//task//children to walk. - Add(psi, "--", "/bin/bash", "-c", command.CommandLine); + Add(psi, "--"); + if (command.DirectExecution is { } directExecution) + { + Add(psi, directExecution.Executable); + Add(psi, [.. directExecution.Arguments]); + } + else + { + Add(psi, "/bin/bash", "-c", command.CommandLine); + } psi.Environment.Clear(); psi.Environment["PATH"] = "/usr/local/bin:/usr/bin:/bin"; diff --git a/packages/Agentweaver.SandboxExec/LinuxBwrapExecutor.cs b/packages/Agentweaver.SandboxExec/LinuxBwrapExecutor.cs index c4814d315..c01c2382c 100644 --- a/packages/Agentweaver.SandboxExec/LinuxBwrapExecutor.cs +++ b/packages/Agentweaver.SandboxExec/LinuxBwrapExecutor.cs @@ -92,6 +92,9 @@ internal static string BuildBwrapPayload(string command, string workdir, bool ne public async Task ExecuteAsync( SandboxCommand command, CancellationToken ct = default) { + if (command.DirectExecution is { } directExecution) + return await ExecuteDirectAsync(command, directExecution, ct).ConfigureAwait(false); + var payload = BuildBwrapPayload( SandboxCommandEnvironment.PrefixPosixExports(command.CommandLine, command.Environment), command.WorkingDirectory, @@ -157,6 +160,119 @@ public async Task ExecuteAsync( } } + private async Task ExecuteDirectAsync( + SandboxCommand command, + SandboxDirectExecution directExecution, + CancellationToken ct) + { + Process? proc = null; + try + { + var psi = BuildDirectProcessStartInfo(command, directExecution); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + if (command.TimeoutMs > 0) + cts.CancelAfter(command.TimeoutMs); + + proc = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start bwrap process."); + + const int stdoutCap = 4 * 1024 * 1024; + const int stderrCap = 1 * 1024 * 1024; + var stdoutTask = ReadBoundedAsync(proc.StandardOutput, stdoutCap, cts.Token); + var stderrTask = ReadBoundedAsync(proc.StandardError, stderrCap, cts.Token); + + try { await proc.WaitForExitAsync(cts.Token).ConfigureAwait(false); } + catch (OperationCanceledException) + { + try { proc.Kill(entireProcessTree: true); } catch { } + throw; + } + + var (stdout, stdoutTrunc) = await stdoutTask.ConfigureAwait(false); + var (stderr, stderrTrunc) = await stderrTask.ConfigureAwait(false); + stdout = SandboxOutputRedactor.Default.Redact(stdout) + .Replace(command.WorkingDirectory, "/workspace", StringComparison.Ordinal); + stderr = SandboxOutputRedactor.Default.Redact(stderr) + .Replace(command.WorkingDirectory, "/workspace", StringComparison.Ordinal); + return new SandboxExecResult(proc.ExitCode, stdout, stderr, false, stdoutTrunc || stderrTrunc); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return new SandboxExecResult(-1, "", "Timed out.", TimedOut: true, OutputTruncated: false); + } + finally + { + if (proc is not null && !proc.HasExited) + try { proc.Kill(entireProcessTree: true); } catch { } + proc?.Dispose(); + } + } + + private static ProcessStartInfo BuildDirectProcessStartInfo( + SandboxCommand command, + SandboxDirectExecution directExecution) + { + var psi = new ProcessStartInfo + { + FileName = "bwrap", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + void Add(params string[] arguments) + { + foreach (var argument in arguments) + psi.ArgumentList.Add(argument); + } + + Add("--bind", command.WorkingDirectory, "/workspace"); + Add( + "--ro-bind-try", "/usr/bin", "/usr/bin", + "--ro-bind-try", "/usr/lib", "/usr/lib", + "--ro-bind-try", "/usr/lib64", "/usr/lib64", + "--ro-bind-try", "/usr/share/nodejs", "/usr/share/nodejs", + "--ro-bind-try", "/usr/local/bin", "/usr/local/bin", + "--ro-bind-try", "/usr/local/lib", "/usr/local/lib", + "--ro-bind-try", "/etc/resolv.conf", "/etc/resolv.conf", + "--ro-bind-try", "/etc/passwd", "/etc/passwd", + "--ro-bind-try", "/etc/group", "/etc/group", + "--ro-bind-try", "/etc/nsswitch.conf", "/etc/nsswitch.conf", + "--symlink", "usr/bin", "/bin", + "--symlink", "usr/lib", "/lib", + "--symlink", "usr/sbin", "/sbin", + "--proc", "/proc", + "--dev", "/dev", + "--tmpfs", "/tmp", + "--tmpfs", "/home", + "--tmpfs", "/root", + "--unshare-pid", + "--unshare-user"); + if (!command.NetworkEnabled) + Add("--unshare-net"); + Add("--new-session", "--clearenv", "--setenv", "PATH", "/usr/local/bin:/usr/bin:/bin"); + AddEnvironment(psi, command.Environment); + AddEnvironment(psi, directExecution.Environment); + Add("--chdir", "/workspace", "--", directExecution.Executable); + foreach (var argument in directExecution.Arguments) + psi.ArgumentList.Add(argument); + return psi; + } + + private static void AddEnvironment( + ProcessStartInfo psi, + IReadOnlyDictionary? environment) + { + if (environment is null) + return; + foreach (var (key, value) in environment) + { + psi.ArgumentList.Add("--setenv"); + psi.ArgumentList.Add(key); + psi.ArgumentList.Add(value); + } + } + public async IAsyncEnumerable StreamAsync( SandboxCommand command, [EnumeratorCancellation] CancellationToken ct = default) diff --git a/packages/Agentweaver.SandboxExec/LinuxNativeMxcSandboxExecutor.cs b/packages/Agentweaver.SandboxExec/LinuxNativeMxcSandboxExecutor.cs index 93148af07..43e3b10e0 100644 --- a/packages/Agentweaver.SandboxExec/LinuxNativeMxcSandboxExecutor.cs +++ b/packages/Agentweaver.SandboxExec/LinuxNativeMxcSandboxExecutor.cs @@ -77,6 +77,16 @@ private static FilesystemPolicy BuildMxcFilesystemPolicy(SandboxFsPolicy policy) public async Task ExecuteAsync( SandboxCommand command, CancellationToken ct = default) { + if (command.DirectExecution is not null) + { + return new SandboxExecResult( + 126, + "", + "Command rejected: this sandbox backend cannot safely deliver repository credentials to a direct process.", + TimedOut: false, + OutputTruncated: false); + } + var mxcPolicy = new SandboxPolicy { Version = "0.4.0-alpha", diff --git a/packages/Agentweaver.SandboxExec/MxcSandboxExecutor.cs b/packages/Agentweaver.SandboxExec/MxcSandboxExecutor.cs index f17ec1012..ee6107888 100644 --- a/packages/Agentweaver.SandboxExec/MxcSandboxExecutor.cs +++ b/packages/Agentweaver.SandboxExec/MxcSandboxExecutor.cs @@ -180,6 +180,16 @@ private FilesystemPolicy BuildMxcFilesystemPolicy(SandboxFsPolicy policy) => public async Task ExecuteAsync( SandboxCommand command, CancellationToken ct = default) { + if (command.DirectExecution is not null) + { + return new SandboxExecResult( + 126, + "", + "Command rejected: this sandbox backend cannot safely deliver repository credentials to a direct process.", + TimedOut: false, + OutputTruncated: false); + } + // Build policy with enrichment (cached at construction). The enrichment provides a // selective tool-path allowlist via PolicyDiscovery.GetAvailableToolsPolicy(), replacing // the broad /usr bind that Copilot CLI does NOT use (Phase 6 alignment). diff --git a/packages/Agentweaver.SandboxExec/PassthroughExecutor.cs b/packages/Agentweaver.SandboxExec/PassthroughExecutor.cs index 01461c874..b7329df53 100644 --- a/packages/Agentweaver.SandboxExec/PassthroughExecutor.cs +++ b/packages/Agentweaver.SandboxExec/PassthroughExecutor.cs @@ -59,7 +59,14 @@ public async Task ExecuteAsync( WorkingDirectory = command.WorkingDirectory, }; - if (OperatingSystem.IsWindows()) + if (command.DirectExecution is { } directExecution) + { + SandboxCommandEnvironment.RemoveInheritedCommandHelperVariables(psi); + psi.FileName = directExecution.Executable; + foreach (var argument in directExecution.Arguments) + psi.ArgumentList.Add(argument); + } + else if (OperatingSystem.IsWindows()) { psi.FileName = "cmd.exe"; psi.ArgumentList.Add("/c"); @@ -72,6 +79,7 @@ public async Task ExecuteAsync( psi.ArgumentList.Add(command.CommandLine); } SandboxCommandEnvironment.ApplyToProcessStartInfo(psi, command.Environment); + SandboxCommandEnvironment.ApplyToProcessStartInfo(psi, command.DirectExecution?.Environment); using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); if (command.TimeoutMs > 0) diff --git a/packages/Agentweaver.SandboxExec/PodExec/PodExecProtocol.cs b/packages/Agentweaver.SandboxExec/PodExec/PodExecProtocol.cs index 4920ec078..92f723584 100644 --- a/packages/Agentweaver.SandboxExec/PodExec/PodExecProtocol.cs +++ b/packages/Agentweaver.SandboxExec/PodExec/PodExecProtocol.cs @@ -57,6 +57,9 @@ public sealed record PodExecRequest public string? Workspace { get; init; } public string? Home { get; init; } public string? CommandLine { get; init; } + public string? DirectExecutable { get; init; } + public List? DirectArguments { get; init; } + public Dictionary? DirectEnvironment { get; init; } public string? WorkingDirectory { get; init; } public Dictionary? Environment { get; init; } public List? ReadWritePaths { get; init; } diff --git a/packages/Agentweaver.SandboxExec/PodExec/PodExecSandboxClient.cs b/packages/Agentweaver.SandboxExec/PodExec/PodExecSandboxClient.cs index 5d89a5060..b21cf3296 100644 --- a/packages/Agentweaver.SandboxExec/PodExec/PodExecSandboxClient.cs +++ b/packages/Agentweaver.SandboxExec/PodExec/PodExecSandboxClient.cs @@ -142,6 +142,12 @@ public async Task ExecuteAsync( { Op = PodExecOps.Exec, CommandLine = command.CommandLine, + DirectExecutable = command.DirectExecution?.Executable, + DirectArguments = command.DirectExecution?.Arguments.ToList(), + DirectEnvironment = command.DirectExecution?.Environment?.ToDictionary( + pair => pair.Key, + pair => pair.Value, + StringComparer.Ordinal), WorkingDirectory = command.WorkingDirectory, Environment = command.Environment?.ToDictionary( pair => pair.Key, diff --git a/packages/Agentweaver.SandboxExec/PodExec/PodExecServer.cs b/packages/Agentweaver.SandboxExec/PodExec/PodExecServer.cs index 45528b89e..a6e6604b7 100644 --- a/packages/Agentweaver.SandboxExec/PodExec/PodExecServer.cs +++ b/packages/Agentweaver.SandboxExec/PodExec/PodExecServer.cs @@ -579,7 +579,13 @@ private SandboxCommand ToCommand(PodExecRequest request) => request.ReadOnlyPaths ?? [], []), request.TimeoutMs, - request.NetworkEnabled); + request.NetworkEnabled, + DirectExecution: string.IsNullOrWhiteSpace(request.DirectExecutable) + ? null + : new SandboxDirectExecution( + request.DirectExecutable, + request.DirectArguments ?? [], + request.DirectEnvironment)); private bool IsAuthorized(string? token) => !string.IsNullOrEmpty(_token) diff --git a/packages/Agentweaver.SandboxExec/SandboxCommandEnvironment.cs b/packages/Agentweaver.SandboxExec/SandboxCommandEnvironment.cs index ddc5c6168..80df073fd 100644 --- a/packages/Agentweaver.SandboxExec/SandboxCommandEnvironment.cs +++ b/packages/Agentweaver.SandboxExec/SandboxCommandEnvironment.cs @@ -16,6 +16,22 @@ public static void ApplyToProcessStartInfo( startInfo.Environment[key] = value; } + public static void RemoveInheritedCommandHelperVariables(ProcessStartInfo startInfo) + { + foreach (var key in startInfo.Environment.Keys.ToArray()) + { + if (key.StartsWith("GIT_", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("GH_", StringComparison.OrdinalIgnoreCase) || + key.StartsWith("GITHUB_", StringComparison.OrdinalIgnoreCase) || + key.Equals("PAGER", StringComparison.OrdinalIgnoreCase) || + key.Equals("EDITOR", StringComparison.OrdinalIgnoreCase) || + key.Equals("VISUAL", StringComparison.OrdinalIgnoreCase)) + { + startInfo.Environment.Remove(key); + } + } + } + public static string PrefixPosixExports( string commandLine, IReadOnlyDictionary? environment) diff --git a/packages/Agentweaver.SandboxExec/WslMxcSandboxExecutor.cs b/packages/Agentweaver.SandboxExec/WslMxcSandboxExecutor.cs index 93d78ddad..e1cdbbfab 100644 --- a/packages/Agentweaver.SandboxExec/WslMxcSandboxExecutor.cs +++ b/packages/Agentweaver.SandboxExec/WslMxcSandboxExecutor.cs @@ -88,6 +88,9 @@ internal WslMxcSandboxExecutor(ILogger logger, ContainmentBackend backend) public async Task ExecuteAsync( SandboxCommand command, CancellationToken ct = default) { + if (command.DirectExecution is { } directExecution) + return await ExecuteDirectAsync(command, directExecution, ct).ConfigureAwait(false); + var commandLine = SandboxCommandEnvironment.PrefixPosixExports( command.CommandLine, command.Environment); @@ -227,6 +230,119 @@ private async Task ExecuteBwrapAsync( } } + private async Task ExecuteDirectAsync( + SandboxCommand command, + SandboxDirectExecution directExecution, + CancellationToken ct) + { + Process? proc = null; + try + { + var psi = BuildDirectProcessStartInfo(command, directExecution); + using var cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + if (command.TimeoutMs > 0) + cts.CancelAfter(command.TimeoutMs); + + proc = Process.Start(psi) + ?? throw new InvalidOperationException("Failed to start wsl.exe process."); + + const int stdoutCap = 4 * 1024 * 1024; + const int stderrCap = 1 * 1024 * 1024; + var stdoutTask = ReadBoundedAsync(proc.StandardOutput, stdoutCap, cts.Token); + var stderrTask = ReadBoundedAsync(proc.StandardError, stderrCap, cts.Token); + try { await proc.WaitForExitAsync(cts.Token).ConfigureAwait(false); } + catch (OperationCanceledException) + { + try { proc.Kill(entireProcessTree: true); } catch { } + throw; + } + + var (stdout, stdoutTrunc) = await stdoutTask.ConfigureAwait(false); + var (stderr, stderrTrunc) = await stderrTask.ConfigureAwait(false); + return new SandboxExecResult( + proc.ExitCode, + SandboxOutputRedactor.Default.Redact(stdout), + SandboxOutputRedactor.Default.Redact(stderr), + false, + stdoutTrunc || stderrTrunc); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + return new SandboxExecResult(-1, "", "Timed out.", TimedOut: true, OutputTruncated: false); + } + finally + { + if (proc is not null && !proc.HasExited) + try { proc.Kill(entireProcessTree: true); } catch { } + proc?.Dispose(); + } + } + + private static ProcessStartInfo BuildDirectProcessStartInfo( + SandboxCommand command, + SandboxDirectExecution directExecution) + { + var psi = new ProcessStartInfo + { + FileName = "wsl.exe", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + void Add(params string[] arguments) + { + foreach (var argument in arguments) + psi.ArgumentList.Add(argument); + } + + Add("--cd", command.WorkingDirectory, "--exec", "bwrap", "--bind", ".", "/workspace"); + Add( + "--ro-bind-try", "/usr/bin", "/usr/bin", + "--ro-bind-try", "/usr/lib", "/usr/lib", + "--ro-bind-try", "/usr/lib64", "/usr/lib64", + "--ro-bind-try", "/usr/share/nodejs", "/usr/share/nodejs", + "--ro-bind-try", "/usr/local/bin", "/usr/local/bin", + "--ro-bind-try", "/usr/local/lib", "/usr/local/lib", + "--ro-bind-try", "/etc/resolv.conf", "/etc/resolv.conf", + "--ro-bind-try", "/etc/passwd", "/etc/passwd", + "--ro-bind-try", "/etc/group", "/etc/group", + "--ro-bind-try", "/etc/nsswitch.conf", "/etc/nsswitch.conf", + "--symlink", "usr/bin", "/bin", + "--symlink", "usr/lib", "/lib", + "--symlink", "usr/sbin", "/sbin", + "--proc", "/proc", + "--dev", "/dev", + "--tmpfs", "/tmp", + "--tmpfs", "/home", + "--tmpfs", "/root", + "--unshare-pid", + "--unshare-user"); + if (!command.NetworkEnabled) + Add("--unshare-net"); + Add("--new-session", "--clearenv", "--setenv", "PATH", "/usr/local/bin:/usr/bin:/bin"); + AddEnvironment(psi, command.Environment); + AddEnvironment(psi, directExecution.Environment); + Add("--chdir", "/workspace", "--", directExecution.Executable); + foreach (var argument in directExecution.Arguments) + psi.ArgumentList.Add(argument); + return psi; + } + + private static void AddEnvironment( + ProcessStartInfo psi, + IReadOnlyDictionary? environment) + { + if (environment is null) + return; + foreach (var (key, value) in environment) + { + psi.ArgumentList.Add("--setenv"); + psi.ArgumentList.Add(key); + psi.ArgumentList.Add(value); + } + } + private static async Task<(string Output, bool Truncated)> ReadBoundedAsync( StreamReader reader, int maxBytes, CancellationToken ct) { diff --git a/tests/Agentweaver.Tests/Auth/TwoAppCredentialArchitectureTests.cs b/tests/Agentweaver.Tests/Auth/TwoAppCredentialArchitectureTests.cs index 2050362cf..d6a103253 100644 --- a/tests/Agentweaver.Tests/Auth/TwoAppCredentialArchitectureTests.cs +++ b/tests/Agentweaver.Tests/Auth/TwoAppCredentialArchitectureTests.cs @@ -128,7 +128,7 @@ public void RepositoryCredentialRegistry_UsesOnlyTheRunBoundRepositorySnapshot() source.Should().Contain("GetCapabilitySnapshotsAsync(runId, ct)") .And.Contain("GitHubCapabilityPurpose.UnattendedRepository") .And.Contain("TryUseRepositoryCredentialAsync") - .And.Contain("ConcurrentDictionary") + .And.Contain("ConcurrentDictionary") .And.NotContain("CommandLine") .And.NotContain("endpoint") .And.NotContain("git ") diff --git a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs index d52fff9c6..857d9739c 100644 --- a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs @@ -1,3 +1,4 @@ +using System.Diagnostics; using Agentweaver.AgentRuntime; using Agentweaver.AgentRuntime.Providers; using Agentweaver.AgentTools; @@ -132,7 +133,7 @@ public async Task Controlled_run_command_rejects_backgrounding_and_destructive_c [Theory] [InlineData("git status")] [InlineData("gh repo view")] - public async Task Controlled_run_command_supplies_repository_credential_only_to_git_and_gh(string command) + public async Task Controlled_run_command_uses_direct_execution_for_repository_credential_commands(string command) { SandboxCommand? observed = null; var executor = new CapturingExecutor(command => observed = command); @@ -145,13 +146,26 @@ await tool.InvokeAsync(new AIFunctionArguments( new Dictionary { ["command"] = command })); observed.Should().NotBeNull(); - observed!.Environment.Should().Contain( - new KeyValuePair("GH_TOKEN", "repository-access-token")); - observed.Environment.Should().Contain( - new KeyValuePair("GITHUB_TOKEN", "repository-access-token")); - observed.Environment.Should().Contain( - new KeyValuePair("GIT_CONFIG_VALUE_0", "!gh auth git-credential")); + observed!.DirectExecution.Should().NotBeNull(); observed.CommandLine.Should().NotContain("repository-access-token"); + observed.Environment.Should().NotContain(pair => pair.Value == "repository-access-token"); + + if (command.StartsWith("git", StringComparison.Ordinal)) + { + observed.DirectExecution!.Executable.Should().Be("git"); + observed.DirectExecution.Environment.Should().BeNull(); + observed.DirectExecution.Arguments.Should().Contain("credential.helper=") + .And.Contain("core.hooksPath=/dev/null") + .And.NotContain("repository-access-token"); + } + else + { + observed.DirectExecution!.Executable.Should().Be("gh"); + observed.DirectExecution.Environment.Should().Contain( + new KeyValuePair("GH_TOKEN", "repository-access-token")); + observed.DirectExecution.Environment.Should().NotContain( + new KeyValuePair("GITHUB_TOKEN", "repository-access-token")); + } } [Fact] @@ -203,10 +217,103 @@ public async Task Controlled_run_command_rejects_compound_credentialed_commands( var result = await tool.InvokeAsync(new AIFunctionArguments( new Dictionary { ["command"] = command })); - result?.ToString().Should().Contain("GitHub credentials require one git or gh command"); + result?.ToString().Should().Contain("direct git or gh command"); + executor.ExecuteCalls.Should().Be(0); + } + + [Theory] + [InlineData("git -c core.hooksPath=/unsafe status")] + [InlineData("git status --config-env=credential.helper=GIT_HELPER")] + [InlineData("git fetch --upload-pack=/bin/sh")] + [InlineData("git status --exec-path=/unsafe")] + public async Task Controlled_run_command_rejects_git_options_that_can_select_helpers_or_configuration(string command) + { + var executor = new CountingExecutor(); + using var tracker = new ShellExecutionTracker(); + var tool = CopilotAIAgent.BuildSessionConfigTools( + BuildContext(executor, tracker, repositoryAccessToken: "repository-access-token"), + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var result = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = command })); + + result?.ToString().Should().Contain("Command rejected"); + executor.ExecuteCalls.Should().Be(0); + } + + [Theory] + [InlineData("gh secret set DEPLOY_KEY --body value")] + [InlineData("gh auth token")] + public async Task Controlled_run_command_requires_operator_approval_for_sensitive_gh_commands(string command) + { + var executor = new CountingExecutor(); + using var tracker = new ShellExecutionTracker(); + var context = BuildContext( + executor, + tracker, + destructivePatterns: [.. SandboxPolicy.Default(_root).DestructiveCommandPatterns], + rejectDestructiveCommands: false); + var tool = CopilotAIAgent.BuildSessionConfigTools( + context, + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var result = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = command })); + + result?.ToString().Should().Contain("requires operator approval"); executor.ExecuteCalls.Should().Be(0); } + [Fact] + public async Task Controlled_run_command_keeps_the_sentinel_out_of_git_alias_hooks_and_helpers() + { + const string sentinel = "sentinel-repository-token"; + var repository = Path.Combine(_root, "credential-boundary"); + var hookOutput = Path.Combine(_root, "hook-token.txt"); + var aliasOutput = Path.Combine(_root, "alias-token.txt"); + var helperOutput = Path.Combine(_root, "helper-token.txt"); + Directory.CreateDirectory(repository); + await RunGitAsync(repository, "init"); + await RunGitAsync(repository, "config", "user.name", "Sandbox Test"); + await RunGitAsync(repository, "config", "user.email", "sandbox@example.invalid"); + var hookPath = Path.Combine(repository, ".git", "hooks", "pre-commit"); + File.WriteAllText( + hookPath, + $"#!/bin/sh{Environment.NewLine}printenv > {PosixQuote(hookOutput)}{Environment.NewLine}"); + if (!OperatingSystem.IsWindows()) + { + File.SetUnixFileMode( + hookPath, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + await RunGitAsync(repository, "config", "alias.steal", $"!printenv > {PosixQuote(aliasOutput)}"); + await RunGitAsync(repository, "config", "credential.helper", $"!printenv > {PosixQuote(helperOutput)}"); + + var executor = SandboxExecutorFactory.CreatePassthrough(); + using var tracker = new ShellExecutionTracker(); + var tool = CopilotAIAgent.BuildSessionConfigTools( + BuildContext( + executor, + tracker, + workspace: repository, + repositoryAccessToken: sentinel), + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var commit = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = "git commit --allow-empty -m credential-boundary" })); + var alias = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = "git steal" })); + var helper = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = "git credential fill" })); + + commit?.ToString().Should().Contain("exit_code: 0"); + alias?.ToString().Should().Contain("built-in git command"); + helper?.ToString().Should().Contain("built-in git command"); + File.Exists(hookOutput).Should().BeFalse(); + File.Exists(aliasOutput).Should().BeFalse(); + File.Exists(helperOutput).Should().BeFalse(); + } + [Fact] public async Task Controlled_run_command_arms_watchdog_deadline_above_executor_timeout_by_grace() { @@ -458,7 +565,8 @@ private SandboxToolContext BuildContext( string? workspace = null, string? scratchDirectory = null, string? repositoryAccessToken = null, - string[]? destructivePatterns = null) => + string[]? destructivePatterns = null, + bool rejectDestructiveCommands = true) => new( AgentId: "agent", WorkingDirectory: workspace ?? _root, @@ -471,7 +579,7 @@ private SandboxToolContext BuildContext( { DestructiveCommandPatterns = destructivePatterns ?? ["rm -rf"], RejectBackgroundCommands = true, - RejectDestructiveCommands = true, + RejectDestructiveCommands = rejectDestructiveCommands, MaximumTimeoutMs = 600_000, RepositoryAccessToken = repositoryAccessToken, }, @@ -479,6 +587,29 @@ private SandboxToolContext BuildContext( ShellExecutionTracker: tracker, ScratchDirectory: scratchDirectory); + private static async Task RunGitAsync(string workingDirectory, params string[] arguments) + { + var startInfo = new ProcessStartInfo + { + FileName = "git", + WorkingDirectory = workingDirectory, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + foreach (var argument in arguments) + startInfo.ArgumentList.Add(argument); + using var process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Could not start git."); + await process.WaitForExitAsync(); + if (process.ExitCode != 0) + throw new InvalidOperationException(await process.StandardError.ReadToEndAsync()); + } + + private static string PosixQuote(string value) => + "'" + value.Replace("'", "'\\''") + "'"; + private sealed class RecordingExecutor(ISandboxExecutor inner) : ISandboxExecutor { public SandboxCommand? LastCommand { get; private set; } diff --git a/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs b/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs new file mode 100644 index 000000000..c76f260cd --- /dev/null +++ b/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs @@ -0,0 +1,67 @@ +using System.Net.Http; +using Agentweaver.Api.Sandbox; +using FluentAssertions; + +namespace Agentweaver.Tests.Sandbox; + +public sealed class RunRepositoryCredentialRegistryTests +{ + [Fact] + public async Task Revoke_RetainsCredentialAfterThrownFailure_ThenRetriesAndCleansUp() + { + var minter = new StubCredentialMinter + { + Credential = new RepositoryCredential( + "registry-sentinel-token", + DateTimeOffset.UtcNow.AddMinutes(5)), + }; + minter.RevokeFailures.Enqueue(new HttpRequestException("GitHub revoke failed.")); + var registry = new RunRepositoryCredentialRegistry(minter); + + (await registry.MintAsync("run-credential-retry")).Should().Be("registry-sentinel-token"); + var first = () => registry.RevokeAsync("run-credential-retry"); + await first.Should().ThrowAsync(); + + await registry.RevokeAsync("run-credential-retry"); + await registry.RevokeAsync("run-credential-retry"); + + minter.RevokedTokens.Should().HaveCount(2, + "a failed revoke must retain its retry state, while a later success removes it"); + minter.RevokedTokens.Should().OnlyContain(token => token == "registry-sentinel-token"); + } + + [Fact] + public async Task Revoke_DropsCredentialOnlyAfterActualExpiry() + { + var minter = new StubCredentialMinter + { + Credential = new RepositoryCredential( + "expired-registry-token", + DateTimeOffset.UtcNow.AddSeconds(-1)), + }; + var registry = new RunRepositoryCredentialRegistry(minter); + + (await registry.MintAsync("run-expired-credential")).Should().BeNull(); + await registry.RevokeAsync("run-expired-credential"); + + minter.RevokedTokens.Should().BeEmpty(); + } + + private sealed class StubCredentialMinter : IRunRepositoryCredentialMinter + { + public RepositoryCredential? Credential { get; init; } + public Queue RevokeFailures { get; } = new(); + public List RevokedTokens { get; } = []; + + public Task MintAsync(string runId, CancellationToken ct) => + Task.FromResult(Credential); + + public Task RevokeAsync(string accessToken, CancellationToken ct) + { + RevokedTokens.Add(accessToken); + if (RevokeFailures.TryDequeue(out var failure)) + throw failure; + return Task.CompletedTask; + } + } +} diff --git a/tests/Agentweaver.Tests/Webhooks/RepoAppInstallationServiceTests.cs b/tests/Agentweaver.Tests/Webhooks/RepoAppInstallationServiceTests.cs index bc4bb5b24..8ecafff4c 100644 --- a/tests/Agentweaver.Tests/Webhooks/RepoAppInstallationServiceTests.cs +++ b/tests/Agentweaver.Tests/Webhooks/RepoAppInstallationServiceTests.cs @@ -123,6 +123,38 @@ public async Task Mint_RejectsDifferentPermissionDigestOrRepositoryBeforeCalling handler.RequestCount.Should().Be(0); } + [Theory] + [InlineData(HttpStatusCode.BadRequest)] + [InlineData(HttpStatusCode.InternalServerError)] + public async Task Revoke_FailsForAnyNonSuccessGitHubResponse(HttpStatusCode statusCode) + { + await using var db = await OpenDbAsync(); + var service = new RepoAppInstallationTokenService( + Config(), + db, + new InMemorySecretStore(), + new StubHttpClientFactory(new StatusHandler(statusCode))); + + var revoke = () => service.RevokeRepositoryTokenAsync("revoke-sentinel"); + + await revoke.Should().ThrowAsync(); + } + + [Fact] + public async Task Revoke_PropagatesThrownProviderFailure() + { + await using var db = await OpenDbAsync(); + var service = new RepoAppInstallationTokenService( + Config(), + db, + new InMemorySecretStore(), + new StubHttpClientFactory(new ThrowingHandler())); + + var revoke = () => service.RevokeRepositoryTokenAsync("revoke-sentinel"); + + await revoke.Should().ThrowAsync(); + } + [Fact] public async Task Lifecycle_ClaimsDeliveriesOnce_AndRoutesOnlyNumericInstallationRepositoryGrant() { @@ -336,7 +368,7 @@ private static async Task OpenDbAsync() return db; } - private sealed class StubHttpClientFactory(RecordingHandler handler) : IHttpClientFactory + private sealed class StubHttpClientFactory(HttpMessageHandler handler) : IHttpClientFactory { public HttpClient CreateClient(string name) => new(handler); } @@ -361,4 +393,16 @@ protected override async Task SendAsync(HttpRequestMessage }; } } + + private sealed class StatusHandler(HttpStatusCode statusCode) : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken ct) => + Task.FromResult(new HttpResponseMessage(statusCode)); + } + + private sealed class ThrowingHandler : HttpMessageHandler + { + protected override Task SendAsync(HttpRequestMessage request, CancellationToken ct) => + throw new HttpRequestException("GitHub transport failed."); + } } From a62a023d325000345d708d5d974695695d16b180 Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 21:08:10 -0700 Subject: [PATCH 04/12] fix: close sandbox repository credential escapes Restrict credential-bearing Git to a child-free command and retry failed token revocations after claim cleanup. Refs #947 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- .../Sandbox/AgentHostReaperService.cs | 18 +++ .../RunRepositoryCredentialRegistry.cs | 149 +++++++++++++++++- docs/deep-dive/sandboxed-execution.md | 14 +- .../Tools/RunCommandTool.cs | 45 ++---- .../Preview/AgentHostReaperCredentialTests.cs | 67 ++++++++ .../AssemblyBuildTestShellGuardTests.cs | 116 ++++++++++---- .../RunRepositoryCredentialRegistryTests.cs | 44 ++++-- 7 files changed, 367 insertions(+), 86 deletions(-) diff --git a/apps/Agentweaver.Api/Sandbox/AgentHostReaperService.cs b/apps/Agentweaver.Api/Sandbox/AgentHostReaperService.cs index b8d8440c0..8a20db8bc 100644 --- a/apps/Agentweaver.Api/Sandbox/AgentHostReaperService.cs +++ b/apps/Agentweaver.Api/Sandbox/AgentHostReaperService.cs @@ -64,6 +64,8 @@ public AgentHostReaperService( /// public async Task SweepOrphanedPodsAsync(CancellationToken ct = default) { + await RetryRetainedRepositoryCredentialRevocationsAsync(ct).ConfigureAwait(false); + var activeMap = await GetActiveClaimMapAsync(ct).ConfigureAwait(false); var claims = await ListAgentHostClaimsAsync(ct).ConfigureAwait(false); var now = DateTimeOffset.UtcNow; @@ -270,6 +272,22 @@ private async Task TryRevokeOrphanRepositoryCredentialAsync(string? runId, Cance } } + private async Task RetryRetainedRepositoryCredentialRevocationsAsync(CancellationToken ct) + { + if (_repositoryCredentials is null) + return; + + var failures = await _repositoryCredentials.RetryFailedRevocationsAsync(ct).ConfigureAwait(false); + foreach (var failure in failures) + { + _logger.LogWarning( + failure.Exception, + "AgentHostReaper: retained repository credential revocation retry failed for run {RunId}; " + + "it will retry with backoff until credential expiry", + failure.RunId); + } + } + /// /// Reconciles every preview-retention side effect before deciding whether to reap. A missing /// service/run id or reconciliation failure defaults to Previewable (leak-safe) rather than diff --git a/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs index ef4d3ef21..97d007ff7 100644 --- a/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs +++ b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs @@ -13,8 +13,14 @@ namespace Agentweaver.Api.Sandbox; /// public sealed class RunRepositoryCredentialRegistry { + internal static readonly TimeSpan InitialRevocationRetryDelay = TimeSpan.FromSeconds(5); + internal static readonly TimeSpan MaximumRevocationRetryDelay = TimeSpan.FromMinutes(1); + private readonly IRunRepositoryCredentialMinter _credentialMinter; + private readonly TimeProvider _timeProvider; private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); + private readonly ConcurrentDictionary _retainedRevocations = + new(StringComparer.Ordinal); private readonly ConcurrentDictionary _mintLocks = new(StringComparer.Ordinal); public RunRepositoryCredentialRegistry(IServiceScopeFactory scopeFactory) @@ -22,8 +28,13 @@ public RunRepositoryCredentialRegistry(IServiceScopeFactory scopeFactory) { } - internal RunRepositoryCredentialRegistry(IRunRepositoryCredentialMinter credentialMinter) => + internal RunRepositoryCredentialRegistry( + IRunRepositoryCredentialMinter credentialMinter, + TimeProvider? timeProvider = null) + { _credentialMinter = credentialMinter; + _timeProvider = timeProvider ?? TimeProvider.System; + } public async Task MintAsync(string runId, CancellationToken ct = default) { @@ -31,15 +42,24 @@ internal RunRepositoryCredentialRegistry(IRunRepositoryCredentialMinter credenti await mintLock.WaitAsync(ct).ConfigureAwait(false); try { + var now = _timeProvider.GetUtcNow(); + if (_retainedRevocations.TryGetValue(runId, out var retained)) + { + if (retained.ExpiresAt > now) + return null; + + _retainedRevocations.TryRemove(runId, out _); + } + if (_entries.TryGetValue(runId, out var current)) { - if (current.ExpiresAt > DateTimeOffset.UtcNow) + if (current.ExpiresAt > now) return null; _entries.TryRemove(runId, out _); } var minted = await _credentialMinter.MintAsync(runId, ct).ConfigureAwait(false); - if (minted is null || minted.ExpiresAt <= DateTimeOffset.UtcNow) + if (minted is null || minted.ExpiresAt <= _timeProvider.GetUtcNow()) return null; _entries[runId] = minted; @@ -60,25 +80,140 @@ public async Task RevokeAsync(string? runId, CancellationToken ct = default) await mintLock.WaitAsync(ct).ConfigureAwait(false); try { - if (!_entries.TryGetValue(runId, out var entry)) + var now = _timeProvider.GetUtcNow(); + if (_retainedRevocations.TryGetValue(runId, out var retained)) + { + if (retained.ExpiresAt <= now) + { + _retainedRevocations.TryRemove(runId, out _); + _entries.TryRemove(runId, out _); + } return; + } - if (entry.ExpiresAt <= DateTimeOffset.UtcNow) + if (!_entries.TryGetValue(runId, out var entry)) + return; + if (entry.ExpiresAt <= now) { _entries.TryRemove(runId, out _); return; } - await _credentialMinter.RevokeAsync(entry.AccessToken, ct).ConfigureAwait(false); - _entries.TryRemove(runId, out _); + await RevokeAndRemoveAsync(runId, entry.AccessToken, entry.ExpiresAt, ct).ConfigureAwait(false); } finally { mintLock.Release(); } } + + /// + /// Retries failed repository-token revocations that are due, including those whose owning + /// SandboxClaim was already deleted. The expiry is an absolute stop: expired tokens are removed + /// without another provider call. + /// + internal async Task> RetryFailedRevocationsAsync( + CancellationToken ct = default) + { + var failures = new List(); + foreach (var runId in _retainedRevocations.Keys) + { + ct.ThrowIfCancellationRequested(); + + var mintLock = _mintLocks.GetOrAdd(runId, static _ => new SemaphoreSlim(1, 1)); + await mintLock.WaitAsync(ct).ConfigureAwait(false); + try + { + if (!_retainedRevocations.TryGetValue(runId, out var retained)) + continue; + + var now = _timeProvider.GetUtcNow(); + if (retained.ExpiresAt <= now) + { + _retainedRevocations.TryRemove(runId, out _); + _entries.TryRemove(runId, out _); + continue; + } + + if (retained.NextAttemptAt > now) + continue; + + try + { + await _credentialMinter.RevokeAsync(retained.AccessToken, ct).ConfigureAwait(false); + _retainedRevocations.TryRemove(runId, out _); + _entries.TryRemove(runId, out _); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + RetainFailedRevocation(runId, retained.AccessToken, retained.ExpiresAt); + failures.Add(new FailedRepositoryCredentialRevocation(runId, ex)); + } + } + finally + { + mintLock.Release(); + } + } + + return failures; + } + + private async Task RevokeAndRemoveAsync( + string runId, + string accessToken, + DateTimeOffset expiresAt, + CancellationToken ct) + { + try + { + await _credentialMinter.RevokeAsync(accessToken, ct).ConfigureAwait(false); + _retainedRevocations.TryRemove(runId, out _); + _entries.TryRemove(runId, out _); + } + catch + { + RetainFailedRevocation(runId, accessToken, expiresAt); + throw; + } + } + + private void RetainFailedRevocation(string runId, string accessToken, DateTimeOffset expiresAt) + { + var failures = _retainedRevocations.TryGetValue(runId, out var retained) + ? retained.FailureCount + 1 + : 1; + var retryDelay = CalculateRetryDelay(failures); + _retainedRevocations[runId] = new RetainedRevocation( + accessToken, + expiresAt, + failures, + _timeProvider.GetUtcNow().Add(retryDelay)); + _entries.TryRemove(runId, out _); + } + + private static TimeSpan CalculateRetryDelay(int failureCount) + { + var multiplier = 1L << Math.Min(Math.Max(failureCount - 1, 0), 6); + var ticks = Math.Min( + InitialRevocationRetryDelay.Ticks * multiplier, + MaximumRevocationRetryDelay.Ticks); + return TimeSpan.FromTicks(ticks); + } + + private sealed record RetainedRevocation( + string AccessToken, + DateTimeOffset ExpiresAt, + int FailureCount, + DateTimeOffset NextAttemptAt); } +internal sealed record FailedRepositoryCredentialRevocation(string RunId, Exception Exception); + /// /// The registry's credential-only dependency. It has no knowledge of git, gh, command text, or /// sandbox execution; it only mints from the run's fenced repository snapshot and revokes the diff --git a/docs/deep-dive/sandboxed-execution.md b/docs/deep-dive/sandboxed-execution.md index 5e5cee92d..beab224d1 100644 --- a/docs/deep-dive/sandboxed-execution.md +++ b/docs/deep-dive/sandboxed-execution.md @@ -86,16 +86,20 @@ The policy is read through `ISandboxPolicyStore.GetPolicyAsync` and is configura The API sends one short-lived installation credential for the selected repository and run. When that credential is used, the sandbox parses the command and starts the approved `git` or `gh` -executable directly instead of placing the credential in a shell environment. Shell syntax, -Git aliases, configured helpers, alternate worktrees, and helper-selecting Git options are -rejected for this path. Git runs with hooks and credential helpers disabled; its GitHub -authorization header is scoped to the directly-started Git process. +executable directly instead of placing the credential in a shell environment. Credential-bearing +Git is limited to argument-free `git status`; all other Git subcommands and flags are rejected. +This prevents repository configuration from selecting external diffs, signing programs, filters, +merge drivers, hooks, aliases, helpers, or remote helpers. The direct Git process disables hooks, +credential helpers, filesystem monitors, and recursive submodules. Its GitHub authorization +header is scoped to that process; it is never supplied to a child process. The approval policy gates `git push`, remote changes, `gh pr` changes, `gh repo` changes, `gh api`, `gh secret set`, and `gh auth` commands (including `gh auth token`). The API does not inspect or proxy these commands. -The system keeps this credential out of pod specs, files, logs, events, annotations, shared environments, and credential-helper files. Normal release and orphan cleanup log failed revocations and retain the in-memory retry state until a later cleanup succeeds or the credential actually expires. +The system keeps this credential out of pod specs, files, logs, events, annotations, shared environments, and credential-helper files. Normal release and orphan cleanup log failed revocations. The +registry retains failed revocations independently of their SandboxClaim, and each reaper sweep retries +due revocations with capped exponential backoff until they succeed or the credential actually expires. ## Security model diff --git a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs index 0c65f5869..2dc6084dc 100644 --- a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs +++ b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs @@ -196,14 +196,7 @@ private static Dictionary BuildCommandEnvironment( return environment; } - private static readonly HashSet BuiltInGitCommands = new(StringComparer.Ordinal) - { - "add", "apply", "blame", "branch", "checkout", "clean", "clone", "commit", - "diff", "fetch", "grep", "log", "ls-files", "ls-remote", "ls-tree", "merge", - "merge-base", "mv", "pull", "push", "rebase", "remote", "reset", "restore", - "revert", "rm", "show", "show-ref", "sparse-checkout", "stash", "status", - "switch", "tag", "worktree", - }; + private const string CredentialSafeGitCommand = "status"; private static readonly HashSet BuiltInGhCommands = new(StringComparer.Ordinal) { @@ -241,6 +234,8 @@ private static bool TryCreateRepositoryCredentialCommand( "--no-pager", "-c", "credential.helper=", "-c", "core.hooksPath=/dev/null", + "-c", "core.fsmonitor=false", + "-c", "submodule.recurse=false", "-c", "protocol.allow=never", "-c", "protocol.https.allow=always", "-c", $"http.https://github.com/.extraheader=AUTHORIZATION: basic {basicAuthorization}", @@ -282,41 +277,19 @@ private static bool TryValidateGitArguments( out string? error) { error = null; - if (arguments.Count < 2 || !BuiltInGitCommands.Contains(arguments[1])) + // Git's built-in command set is not a safety boundary. Many built-ins can invoke + // repository-configured helpers (diffs, filters, merge drivers, signing, and hooks), and + // Git forwards -c configuration to those children. Keep the credential path to the one + // argument-free inspection command for which Git starts no repository-controlled child. + if (arguments.Count != 2 || arguments[1] != CredentialSafeGitCommand) { - error = "Command rejected: repository credentials require a built-in git command."; - return false; - } - - foreach (var argument in arguments.Skip(2)) - { - if (!IsUnsafeGitArgument(argument)) - continue; - - error = "Command rejected: git configuration, aliases, helpers, and alternate worktrees are not allowed with repository credentials."; + error = "Command rejected: repository credentials only allow 'git status' without arguments."; return false; } return true; } - private static bool IsUnsafeGitArgument(string argument) => - argument == "-c" || - argument.StartsWith("-c", StringComparison.Ordinal) || - argument == "-C" || - argument.StartsWith("--config", StringComparison.Ordinal) || - argument.StartsWith("--exec-path", StringComparison.Ordinal) || - argument.StartsWith("--git-dir", StringComparison.Ordinal) || - argument.StartsWith("--work-tree", StringComparison.Ordinal) || - argument.StartsWith("--namespace", StringComparison.Ordinal) || - argument.StartsWith("--upload-pack", StringComparison.Ordinal) || - argument.StartsWith("--receive-pack", StringComparison.Ordinal) || - argument.StartsWith("--git-upload-pack", StringComparison.Ordinal) || - argument.StartsWith("--git-receive-pack", StringComparison.Ordinal) || - argument.StartsWith("--paginate", StringComparison.Ordinal) || - argument == "-p" || - argument.StartsWith("ext::", StringComparison.OrdinalIgnoreCase); - private static bool TryValidateGhArguments( IReadOnlyList arguments, out string? error) diff --git a/tests/Agentweaver.Tests/Preview/AgentHostReaperCredentialTests.cs b/tests/Agentweaver.Tests/Preview/AgentHostReaperCredentialTests.cs index 4b0a4c52d..6e85a9224 100644 --- a/tests/Agentweaver.Tests/Preview/AgentHostReaperCredentialTests.cs +++ b/tests/Agentweaver.Tests/Preview/AgentHostReaperCredentialTests.cs @@ -100,6 +100,55 @@ public async Task Sweep_OrphanClaim_WithoutRunIdAnnotation_DoesNotThrow_AndStill reaped.Should().Be(1); // claim still reaped; credential delete is a best-effort no-op } + [Fact] + public async Task Sweep_OrphanCleanup_RetriesRetainedRepositoryRevocationAfterClaimDeletion() + { + const string runId = "run-retry-after-orphan-cleanup"; + const string accessToken = "reaper-retry-sentinel-token"; + var now = new DateTimeOffset(2026, 8, 27, 20, 0, 0, TimeSpan.Zero); + var clock = new MutableTimeProvider(now); + var minter = new StubCredentialMinter + { + Credential = new RepositoryCredential(accessToken, now.AddMinutes(5)), + }; + minter.RevokeFailures.Enqueue(new HttpRequestException("temporary GitHub revoke failure")); + var registry = new RunRepositoryCredentialRegistry(minter, clock); + (await registry.MintAsync(runId)).Should().Be(accessToken); + + var claimName = SandboxClaimConventions.DeriveAgentHostClaimName(runId); + var initialHandler = new FakeKubeHandler(); + initialHandler.OnGet(ListPath, ClaimsListJson(claimName, runId)); + var initialReaper = new AgentHostReaperService( + ClientFor(initialHandler), + new EmptyRunStore(), + new KubernetesSandboxOptions { Namespace = Namespace }, + NullLogger.Instance, + repositoryCredentials: registry); + + (await initialReaper.SweepOrphanedPodsAsync()).Should().Be(1); + initialHandler.Requests.Should().Contain(request => + request.Method == "DELETE" && request.Path.EndsWith($"/sandboxclaims/{claimName}")); + minter.RevokedTokens.Should().ContainSingle().Which.Should().Be(accessToken); + + var retryHandler = new FakeKubeHandler(); + retryHandler.OnGet(ListPath, """{"items":[]}"""); + var retryReaper = new AgentHostReaperService( + ClientFor(retryHandler), + new EmptyRunStore(), + new KubernetesSandboxOptions { Namespace = Namespace }, + NullLogger.Instance, + repositoryCredentials: registry); + + (await retryReaper.SweepOrphanedPodsAsync()).Should().Be(0); + minter.RevokedTokens.Should().ContainSingle("the retry honors its initial backoff"); + + clock.Advance(RunRepositoryCredentialRegistry.InitialRevocationRetryDelay); + (await retryReaper.SweepOrphanedPodsAsync()).Should().Be(0); + minter.RevokedTokens.Should().HaveCount(2, + "the reaper retries retained state even though the original orphaned claim is gone"); + minter.RevokedTokens.Should().OnlyContain(token => token == accessToken); + } + // Issue #542: a completed subtask's claim is an "orphan" per the active-run map the instant its // turn ends. The reaper must NOT reap it while the run still has a live preview (that would 404 the // preview URL), but MUST reap it once no preview is active (bounded eventual teardown — no leak). @@ -368,6 +417,24 @@ private sealed class MutableTimeProvider : TimeProvider public void Advance(TimeSpan duration) => _utcNow += duration; } + private sealed class StubCredentialMinter : IRunRepositoryCredentialMinter + { + public RepositoryCredential? Credential { get; init; } + public Queue RevokeFailures { get; } = new(); + public List RevokedTokens { get; } = []; + + public Task MintAsync(string runId, CancellationToken ct) => + Task.FromResult(Credential); + + public Task RevokeAsync(string accessToken, CancellationToken ct) + { + RevokedTokens.Add(accessToken); + if (RevokeFailures.TryDequeue(out var failure)) + throw failure; + return Task.CompletedTask; + } + } + // Minimal ISandboxPreviewService test double for the reaper defer path (#542): only lifecycle // reconciliation is consulted; every other member throws so an unexpected call is loud. private sealed class StubPreviewService : ISandboxPreviewService diff --git a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs index 857d9739c..a16c258b3 100644 --- a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs @@ -156,6 +156,8 @@ await tool.InvokeAsync(new AIFunctionArguments( observed.DirectExecution.Environment.Should().BeNull(); observed.DirectExecution.Arguments.Should().Contain("credential.helper=") .And.Contain("core.hooksPath=/dev/null") + .And.Contain("core.fsmonitor=false") + .And.Contain("submodule.recurse=false") .And.NotContain("repository-access-token"); } else @@ -241,6 +243,31 @@ public async Task Controlled_run_command_rejects_git_options_that_can_select_hel executor.ExecuteCalls.Should().Be(0); } + [Theory] + [InlineData("git status --short")] + [InlineData("git diff --ext-diff")] + [InlineData("git commit -S -m signed")] + [InlineData("git add .")] + [InlineData("git merge main")] + [InlineData("git config --get alias.steal")] + [InlineData("git credential fill")] + [InlineData("git steal")] + [InlineData("git fetch origin")] + public async Task Controlled_run_command_rejects_git_commands_that_can_start_child_programs(string command) + { + var executor = new CountingExecutor(); + using var tracker = new ShellExecutionTracker(); + var tool = CopilotAIAgent.BuildSessionConfigTools( + BuildContext(executor, tracker, repositoryAccessToken: "repository-access-token"), + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var result = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = command })); + + result?.ToString().Should().Contain("only allow 'git status' without arguments"); + executor.ExecuteCalls.Should().Be(0); + } + [Theory] [InlineData("gh secret set DEPLOY_KEY --body value")] [InlineData("gh auth token")] @@ -265,29 +292,38 @@ public async Task Controlled_run_command_requires_operator_approval_for_sensitiv } [Fact] - public async Task Controlled_run_command_keeps_the_sentinel_out_of_git_alias_hooks_and_helpers() + public async Task Controlled_run_command_does_not_leak_the_sentinel_to_a_repository_configured_external_diff() { const string sentinel = "sentinel-repository-token"; var repository = Path.Combine(_root, "credential-boundary"); - var hookOutput = Path.Combine(_root, "hook-token.txt"); - var aliasOutput = Path.Combine(_root, "alias-token.txt"); - var helperOutput = Path.Combine(_root, "helper-token.txt"); + var observerOutput = Path.Combine(_root, "external-diff-token.txt"); + var observerExecutable = Path.Combine(repository, "credential-observer.sh"); Directory.CreateDirectory(repository); await RunGitAsync(repository, "init"); await RunGitAsync(repository, "config", "user.name", "Sandbox Test"); await RunGitAsync(repository, "config", "user.email", "sandbox@example.invalid"); - var hookPath = Path.Combine(repository, ".git", "hooks", "pre-commit"); - File.WriteAllText( - hookPath, - $"#!/bin/sh{Environment.NewLine}printenv > {PosixQuote(hookOutput)}{Environment.NewLine}"); - if (!OperatingSystem.IsWindows()) - { - File.SetUnixFileMode( - hookPath, - UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); - } - await RunGitAsync(repository, "config", "alias.steal", $"!printenv > {PosixQuote(aliasOutput)}"); - await RunGitAsync(repository, "config", "credential.helper", $"!printenv > {PosixQuote(helperOutput)}"); + var trackedFile = Path.Combine(repository, "tracked.txt"); + File.WriteAllText(trackedFile, "before"); + await RunGitAsync(repository, "add", "tracked.txt"); + await RunGitAsync(repository, "commit", "-m", "initial"); + File.WriteAllText(trackedFile, "after"); + WriteEnvironmentObserver(observerExecutable, observerOutput); + var externalDiffCommand = OperatingSystem.IsWindows() + ? $"sh {PosixQuote(ToGitShellPath(observerExecutable))}" + : observerExecutable; + await RunGitAsync(repository, "config", "diff.external", externalDiffCommand); + var basicAuthorization = Convert.ToBase64String( + System.Text.Encoding.UTF8.GetBytes($"x-access-token:{sentinel}")); + await RunGitAsync( + repository, + "-c", + $"http.https://github.com/.extraheader=AUTHORIZATION: basic {basicAuthorization}", + "diff", + "--ext-diff"); + File.ReadAllText(observerOutput).Should().Contain(basicAuthorization, + "an unprotected external diff would receive Git configuration containing the sentinel authorization"); + File.Delete(observerOutput); + await RunGitAsync(repository, "config", "core.fsmonitor", externalDiffCommand); var executor = SandboxExecutorFactory.CreatePassthrough(); using var tracker = new ShellExecutionTracker(); @@ -299,19 +335,17 @@ public async Task Controlled_run_command_keeps_the_sentinel_out_of_git_alias_hoo repositoryAccessToken: sentinel), includeControlledRunCommand: true).Single(t => t.Name == "run_command"); - var commit = await tool.InvokeAsync(new AIFunctionArguments( - new Dictionary { ["command"] = "git commit --allow-empty -m credential-boundary" })); - var alias = await tool.InvokeAsync(new AIFunctionArguments( - new Dictionary { ["command"] = "git steal" })); - var helper = await tool.InvokeAsync(new AIFunctionArguments( - new Dictionary { ["command"] = "git credential fill" })); - - commit?.ToString().Should().Contain("exit_code: 0"); - alias?.ToString().Should().Contain("built-in git command"); - helper?.ToString().Should().Contain("built-in git command"); - File.Exists(hookOutput).Should().BeFalse(); - File.Exists(aliasOutput).Should().BeFalse(); - File.Exists(helperOutput).Should().BeFalse(); + var diff = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = "git diff --ext-diff" })); + var status = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = "git status" })); + + diff?.ToString().Should().Contain("only allow 'git status' without arguments"); + File.Exists(observerOutput).Should().BeFalse( + "the credential-bearing command must be rejected before its repository-configured child can start"); + status?.ToString().Should().Contain("exit_code: 0"); + File.Exists(observerOutput).Should().BeFalse( + "the allowed credential-bearing status command disables repository-configured filesystem monitors"); } [Fact] @@ -610,6 +644,30 @@ private static async Task RunGitAsync(string workingDirectory, params string[] a private static string PosixQuote(string value) => "'" + value.Replace("'", "'\\''") + "'"; + private static void WriteEnvironmentObserver(string executable, string outputPath) + { + File.WriteAllText( + executable, + $"#!/bin/sh{Environment.NewLine}printenv > {PosixQuote(ToGitShellPath(outputPath))}{Environment.NewLine}"); + if (!OperatingSystem.IsWindows()) + { + File.SetUnixFileMode( + executable, + UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + } + } + + private static string ToGitShellPath(string path) + { + if (!OperatingSystem.IsWindows()) + return path; + + var fullPath = Path.GetFullPath(path).Replace('\\', '/'); + return fullPath.Length >= 3 && fullPath[1] == ':' + ? $"/{char.ToLowerInvariant(fullPath[0])}{fullPath[2..]}" + : fullPath; + } + private sealed class RecordingExecutor(ISandboxExecutor inner) : ISandboxExecutor { public SandboxCommand? LastCommand { get; private set; } diff --git a/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs b/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs index c76f260cd..f18567df2 100644 --- a/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs @@ -9,42 +9,59 @@ public sealed class RunRepositoryCredentialRegistryTests [Fact] public async Task Revoke_RetainsCredentialAfterThrownFailure_ThenRetriesAndCleansUp() { + var now = new DateTimeOffset(2026, 8, 27, 20, 0, 0, TimeSpan.Zero); var minter = new StubCredentialMinter { Credential = new RepositoryCredential( "registry-sentinel-token", - DateTimeOffset.UtcNow.AddMinutes(5)), + now.AddMinutes(5)), }; minter.RevokeFailures.Enqueue(new HttpRequestException("GitHub revoke failed.")); - var registry = new RunRepositoryCredentialRegistry(minter); + var clock = new MutableTimeProvider(now); + var registry = new RunRepositoryCredentialRegistry(minter, clock); (await registry.MintAsync("run-credential-retry")).Should().Be("registry-sentinel-token"); var first = () => registry.RevokeAsync("run-credential-retry"); await first.Should().ThrowAsync(); await registry.RevokeAsync("run-credential-retry"); - await registry.RevokeAsync("run-credential-retry"); + (await registry.RetryFailedRevocationsAsync()).Should().BeEmpty( + "the automatic retry must wait for its initial backoff"); + minter.RevokedTokens.Should().ContainSingle(); + + clock.Advance(RunRepositoryCredentialRegistry.InitialRevocationRetryDelay); + (await registry.RetryFailedRevocationsAsync()).Should().BeEmpty(); + await registry.RetryFailedRevocationsAsync(); minter.RevokedTokens.Should().HaveCount(2, - "a failed revoke must retain its retry state, while a later success removes it"); + "a failed revoke must retain its retry state, while a later automatic success removes it"); minter.RevokedTokens.Should().OnlyContain(token => token == "registry-sentinel-token"); } [Fact] - public async Task Revoke_DropsCredentialOnlyAfterActualExpiry() + public async Task Retry_DropsRetainedCredentialOnlyAfterActualExpiry() { + var now = new DateTimeOffset(2026, 8, 27, 20, 0, 0, TimeSpan.Zero); var minter = new StubCredentialMinter { Credential = new RepositoryCredential( "expired-registry-token", - DateTimeOffset.UtcNow.AddSeconds(-1)), + now.AddSeconds(1)), }; - var registry = new RunRepositoryCredentialRegistry(minter); + minter.RevokeFailures.Enqueue(new HttpRequestException("GitHub revoke failed.")); + var clock = new MutableTimeProvider(now); + var registry = new RunRepositoryCredentialRegistry(minter, clock); + + (await registry.MintAsync("run-expired-credential")).Should().Be("expired-registry-token"); + var first = () => registry.RevokeAsync("run-expired-credential"); + await first.Should().ThrowAsync(); - (await registry.MintAsync("run-expired-credential")).Should().BeNull(); + clock.Advance(RunRepositoryCredentialRegistry.InitialRevocationRetryDelay); await registry.RevokeAsync("run-expired-credential"); + await registry.RetryFailedRevocationsAsync(); - minter.RevokedTokens.Should().BeEmpty(); + minter.RevokedTokens.Should().ContainSingle( + "an expired credential must not be sent for another provider revocation attempt"); } private sealed class StubCredentialMinter : IRunRepositoryCredentialMinter @@ -64,4 +81,13 @@ public Task RevokeAsync(string accessToken, CancellationToken ct) return Task.CompletedTask; } } + + private sealed class MutableTimeProvider(DateTimeOffset utcNow) : TimeProvider + { + private DateTimeOffset _utcNow = utcNow; + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan duration) => _utcNow += duration; + } } From 183086aa232a7f97354fac13bb643a09c538f0c9 Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 21:11:30 -0700 Subject: [PATCH 05/12] fix: restore canonical sandbox approval defaults Use the canonical destructive command list when sandbox YAML omits it, while preserving explicit policy overrides. Refs #947 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- .../Infrastructure/YamlSandboxPolicyStore.cs | 58 ++------ docs/deep-dive/sandboxed-execution.md | 3 + .../Sandbox/SandboxPolicyPreserveTests.cs | 139 ++++++++++++++++++ 3 files changed, 153 insertions(+), 47 deletions(-) diff --git a/apps/Agentweaver.Api/Infrastructure/YamlSandboxPolicyStore.cs b/apps/Agentweaver.Api/Infrastructure/YamlSandboxPolicyStore.cs index 90fab2f1e..208e25278 100644 --- a/apps/Agentweaver.Api/Infrastructure/YamlSandboxPolicyStore.cs +++ b/apps/Agentweaver.Api/Infrastructure/YamlSandboxPolicyStore.cs @@ -43,10 +43,11 @@ public Task GetPolicyAsync(string repositoryPath, CancellationTok { var yaml = File.ReadAllText(filePath); var root = Deserializer.Deserialize>(yaml) ?? new(StringComparer.OrdinalIgnoreCase); - SandboxPolicyYamlDto dto = new(); - if (TryGetValue(root, "sandbox", out var sandboxNode) && sandboxNode is not null) - dto = Deserializer.Deserialize(Serializer.Serialize(sandboxNode)) ?? new(); - return Task.FromResult(dto.ToDomain(repositoryPath)); + if (!TryGetValue(root, "sandbox", out var sandboxNode) || sandboxNode is null) + return Task.FromResult(SandboxPolicy.Default(repositoryPath)); + + var dto = Deserializer.Deserialize(Serializer.Serialize(sandboxNode)) ?? new(); + return Task.FromResult(dto.ToDomain(SandboxPolicy.Default(repositoryPath))); } catch (Exception ex) when (ex is YamlDotNet.Core.YamlException or IOException) { @@ -136,58 +137,21 @@ internal sealed class SandboxPolicyYamlDto public bool NetworkEnabled { get; set; } = true; public List AllowedRepositoryRoots { get; set; } = []; - public List DestructiveCommandPatterns { get; set; } = - [ - // File deletion - "rm -rf", "rm -fr", "rm -r /", "shred ", "wipe ", - "find / -delete", "find / -exec rm", - "truncate --size 0", - // Disk and filesystem - "dd if=", "mkfs", "fdisk", "parted ", "wipefs", - "> /dev/sd", "> /dev/hd", "> /dev/nvme", - // Windows CMD destructive - "del /s", "rd /s /q", "format ", "cipher /w", - // Privilege escalation and system accounts - "chmod -R 777", "chmod -R 0777", "chown -R root", - "sudo rm", "sudo mkfs", "sudo dd", - "passwd ", "visudo", - // Process and system control - "kill -9", "pkill -9", "killall ", - "shutdown ", "reboot", "halt", "poweroff", "init 0", "init 6", - "systemctl stop", "systemctl disable", "service stop", - // Remote code execution from internet - "curl | sh", "curl | bash", "wget | sh", "wget | bash", - "bash <(curl", "sh <(curl", "eval $(curl", "eval $(wget", - "| bash", "| sh", - // Git destructive - "git push --force", "git push -f", - "git reset --hard", - "git push origin --delete", "git push --delete", - "git branch -D", - "git clean -fd", "git clean -fxd", - // GitHub credential commands - "gh secret set", "gh auth token", - // PowerShell destructive - "Remove-Item -Recurse", "Remove-Item -Force", "ri -r", "ri -Recurse", - "Format-Volume", "Clear-Disk", - "Stop-Process -Force", - "Set-ExecutionPolicy Unrestricted", "Set-ExecutionPolicy Bypass", - "Invoke-Expression", "iex ", - "[System.IO.File]::Delete", - "Get-ChildItem | Remove-Item", - ]; + public List? DestructiveCommandPatterns { get; set; } public bool RequireApprovalForAllShell { get; set; } = false; public bool RedactPii { get; set; } = true; public int MaxOutputBytes { get; set; } = 4 * 1024 * 1024; - public SandboxPolicy ToDomain(string repositoryPath) => new() + public SandboxPolicy ToDomain(SandboxPolicy defaults) => new() { - RepositoryPath = repositoryPath, + RepositoryPath = defaults.RepositoryPath, ShellEnabled = ShellEnabled, Direct = Direct, NetworkEnabled = NetworkEnabled, AllowedRepositoryRoots = AllowedRepositoryRoots, - DestructiveCommandPatterns = DestructiveCommandPatterns, + DestructiveCommandPatterns = DestructiveCommandPatterns is null + ? [.. defaults.DestructiveCommandPatterns] + : DestructiveCommandPatterns, RequireApprovalForAllShell = RequireApprovalForAllShell, RedactPii = RedactPii, MaxOutputBytes = MaxOutputBytes, diff --git a/docs/deep-dive/sandboxed-execution.md b/docs/deep-dive/sandboxed-execution.md index beab224d1..ec2e54b8c 100644 --- a/docs/deep-dive/sandboxed-execution.md +++ b/docs/deep-dive/sandboxed-execution.md @@ -83,6 +83,9 @@ The API (`GET /api/sandbox-policy`, `PUT /api/sandbox-policy`) reads and writes | `MaxOutputBytes` | `int` | `4194304` (4 MB) | Output cap. Results exceeding this are truncated and marked with `OutputTruncated: true`. | The policy is read through `ISandboxPolicyStore.GetPolicyAsync` and is configurable via the API at `GET /api/sandbox-policy` and `PUT /api/sandbox-policy`. See [sandbox-setup.md](../reference/sandbox-setup.md) for operator instructions. +When the settings file has no `sandbox` section, or its `sandbox` section omits +`destructive_command_patterns`, the canonical default approval patterns apply. An explicit list, +including `[]`, is an intentional override. The API sends one short-lived installation credential for the selected repository and run. When that credential is used, the sandbox parses the command and starts the approved `git` or `gh` diff --git a/tests/Agentweaver.Tests/Sandbox/SandboxPolicyPreserveTests.cs b/tests/Agentweaver.Tests/Sandbox/SandboxPolicyPreserveTests.cs index 41b2f4de6..fc6b621d5 100644 --- a/tests/Agentweaver.Tests/Sandbox/SandboxPolicyPreserveTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/SandboxPolicyPreserveTests.cs @@ -3,9 +3,17 @@ using System.Net.Http.Json; using System.Text.Json; using FluentAssertions; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging.Abstractions; +using Agentweaver.AgentRuntime; +using Agentweaver.AgentTools; +using Agentweaver.Api.Infrastructure; using Agentweaver.Api.Contracts; using Agentweaver.Api.Endpoints; using Agentweaver.Domain; +using Agentweaver.SandboxExec; +using Agentweaver.SandboxFs; using Agentweaver.Tests.Helpers; namespace Agentweaver.Tests.Sandbox; @@ -179,6 +187,56 @@ public async Task Put_MissingRepositoryPath_Returns400() resp.StatusCode.Should().Be(HttpStatusCode.BadRequest); } + [Fact] + public async Task GetPolicy_FileWithoutSandboxSection_UsesCanonicalApprovalPatterns() + { + await WriteSettingsAsync( + """ + project: + name: policy-test + """); + + var policy = await _factory.Services.GetRequiredService() + .GetPolicyAsync(_repoPath); + + AssertCanonicalPatternDefaults(policy, _repoPath); + await AssertCanonicalCommandsRequireApprovalAsync(policy); + } + + [Fact] + public async Task GetPolicy_SandboxWithoutDestructivePatterns_UsesCanonicalApprovalPatterns() + { + await WriteSettingsAsync( + """ + sandbox: + shell_enabled: true + network_enabled: false + """); + + var policy = await _factory.Services.GetRequiredService() + .GetPolicyAsync(_repoPath); + + policy.NetworkEnabled.Should().BeFalse(); + AssertCanonicalPatternDefaults(policy, _repoPath); + await AssertCanonicalCommandsRequireApprovalAsync(policy); + } + + [Fact] + public async Task GetPolicy_ExplicitDestructivePatterns_RemainsAnIntentionalOverride() + { + await WriteSettingsAsync( + """ + sandbox: + destructive_command_patterns: + - user-defined-command + """); + + var policy = await _factory.Services.GetRequiredService() + .GetPolicyAsync(_repoPath); + + policy.DestructiveCommandPatterns.Should().Equal("user-defined-command"); + } + // ── Helpers ───────────────────────────────────────────────────────────────────────────────── private async Task SeedFullPolicyAsync() @@ -220,4 +278,85 @@ private static void AssertSeededFieldsPreserved(JsonElement body) private static string[] Roots(JsonElement body, string property) => body.GetProperty(property).EnumerateArray().Select(e => e.GetString()!).ToArray(); + + private static void AssertCanonicalPatternDefaults(SandboxPolicy policy, string repositoryPath) => + policy.DestructiveCommandPatterns.Should().Equal( + SandboxPolicy.Default(repositoryPath).DestructiveCommandPatterns); + + private Task WriteSettingsAsync(string yaml) + { + var settingsDirectory = Path.Combine(_repoPath, ".agentweaver"); + Directory.CreateDirectory(settingsDirectory); + File.WriteAllText(Path.Combine(settingsDirectory, "settings.yml"), yaml); + return Task.CompletedTask; + } + + private static async Task AssertCanonicalCommandsRequireApprovalAsync(SandboxPolicy policy) + { + var executor = new ApprovalRequiredExecutor(); + using var tracker = new ShellExecutionTracker(); + var context = new SandboxToolContext( + AgentId: "agent", + WorkingDirectory: policy.RepositoryPath, + SandboxRoot: policy.RepositoryPath, + Executor: executor, + FileTools: new SandboxedFileTools(policy.RepositoryPath), + SearchTools: new SandboxedSearchTools(policy.RepositoryPath), + Redactor: SandboxOutputRedactor.Default, + Options: new SandboxToolOptions(ShellEnabled: true) + { + DestructiveCommandPatterns = [.. policy.DestructiveCommandPatterns], + }, + Logger: NullLogger.Instance, + ShellExecutionTracker: tracker); + var tool = CopilotAIAgent.BuildSessionConfigTools( + context, + includeControlledRunCommand: true).Single(tool => tool.Name == "run_command"); + + foreach (var command in new[] + { + "gh api /user", + "git push origin main", + "gh auth login", + "gh auth logout", + "gh pr create --title test --body test", + "gh pr merge 1", + "gh pr close 1", + "gh repo delete example/repo", + "gh repo archive example/repo", + }) + { + var result = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = command })); + result?.ToString().Should().Contain("requires operator approval"); + } + + executor.ExecuteCalls.Should().Be(0); + } + + private sealed class ApprovalRequiredExecutor : ISandboxExecutor + { + public int ExecuteCalls { get; private set; } + public bool IsRealIsolation => true; + public string BackendName => "test"; + public string SelectionReason => "test"; + public bool HasNetworkWarning => false; + public string? NetworkWarningMessage => null; + + public Task ExecuteAsync( + SandboxCommand command, + CancellationToken ct = default) + { + ExecuteCalls++; + return Task.FromResult(new SandboxExecResult(0, "", "", false, false)); + } + + public async IAsyncEnumerable StreamAsync( + SandboxCommand command, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct = default) + { + await Task.CompletedTask; + yield break; + } + } } From f12e7c18bf2e7c31642577a5296bbff744aece5a Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 21:23:40 -0700 Subject: [PATCH 06/12] fix: harden credentialed gh command approval Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- .../Tools/RunCommandTool.cs | 120 +++++++++++++--- .../AssemblyBuildTestShellGuardTests.cs | 132 +++++++++++++++++- 2 files changed, 233 insertions(+), 19 deletions(-) diff --git a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs index 2dc6084dc..29121e40a 100644 --- a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs +++ b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs @@ -19,12 +19,28 @@ public AIFunction CreateFunction(SandboxToolContext ctx) => if (ctx.Options.RejectBackgroundCommands && ContainsBackgrounding(command)) return "Command rejected: background/detached shell execution is not allowed."; - var destructive = IsDestructivePattern(command, ctx.Options.DestructiveCommandPatterns); + IReadOnlyList? credentialArguments = null; + var approvalCommand = command; + var commandHash = ComputeCommandHash(command); + if (!string.IsNullOrWhiteSpace(ctx.Options.RepositoryAccessToken)) + { + if (!TryParseCommand(command, out credentialArguments, out var credentialParseError)) + return credentialParseError!; + + if (credentialArguments.Count > 0 && + (credentialArguments[0] == "git" || credentialArguments[0] == "gh")) + { + // Approval must describe and identify the exact argv that receives the + // credential, rather than the shell spelling the model originally sent. + approvalCommand = FormatParsedCommand(credentialArguments); + commandHash = ComputeCommandHash(credentialArguments); + } + } + + var destructive = IsDestructivePattern(approvalCommand, ctx.Options.DestructiveCommandPatterns); if (ctx.Options.RejectDestructiveCommands && destructive) return "Command rejected: destructive shell commands are not allowed in the Build/Test gate."; - var commandHash = ComputeCommandHash(command); - // HITL gate: destructive commands require operator approval before execution. if (ctx.Options.RequireApprovalForAllShell || destructive) { @@ -100,7 +116,7 @@ public AIFunction CreateFunction(SandboxToolContext ctx) => if (ctx.Options.MaximumTimeoutMs > 0) timeout = Math.Min(timeout, ctx.Options.MaximumTimeoutMs); if (!TryCreateRepositoryCredentialCommand( - command, + credentialArguments, ctx.Options.RepositoryAccessToken, out var directExecution, out var credentialError)) @@ -156,6 +172,11 @@ private static string ComputeCommandHash(string command) => System.Security.Cryptography.SHA256.HashData( System.Text.Encoding.UTF8.GetBytes(command)))[..16].ToLowerInvariant(); + private static string ComputeCommandHash(IReadOnlyList arguments) => + // TryParseCommand rejects NUL, so this separator gives every parsed argv sequence + // a distinct stable identity without returning to the shell's original text. + ComputeCommandHash(string.Join('\0', arguments)); + private static string? ResolveScratchDirectory(SandboxToolContext ctx) { if (!string.IsNullOrWhiteSpace(ctx.ScratchDirectory)) @@ -206,7 +227,7 @@ private static Dictionary BuildCommandEnvironment( }; private static bool TryCreateRepositoryCredentialCommand( - string command, + IReadOnlyList? arguments, string? accessToken, out SandboxDirectExecution? directExecution, out string? error) @@ -216,8 +237,8 @@ private static bool TryCreateRepositoryCredentialCommand( if (string.IsNullOrWhiteSpace(accessToken)) return true; - if (!TryParseCommand(command, out var arguments, out error)) - return false; + if (arguments is null) + throw new InvalidOperationException("Credential-bearing command arguments must be parsed before execution."); if (arguments.Count == 0 || (arguments[0] != "git" && arguments[0] != "gh")) return true; @@ -301,15 +322,7 @@ private static bool TryValidateGhArguments( return false; } - var hasNestedCommand = (string topLevel, string nested) => - arguments[1] == topLevel && - arguments.Skip(2).Any(argument => argument == nested); - if (arguments[1] == "codespace" || - arguments.Skip(2).Any(argument => - argument is "--web" or "--browser" or "--editor") || - hasNestedCommand("auth", "setup-git") || - hasNestedCommand("repo", "clone") || - hasNestedCommand("pr", "checkout")) + if (GhArgumentsCanStartAnotherExecutable(arguments)) { error = "Command rejected: gh commands that start another executable are not allowed with repository credentials."; return false; @@ -318,6 +331,58 @@ private static bool TryValidateGhArguments( return true; } + private static bool GhArgumentsCanStartAnotherExecutable(IReadOnlyList arguments) + { + if (arguments[1] == "codespace" || HasGhProcessLaunchingOption(arguments)) + return true; + + return HasNestedGhCommand(arguments, "auth", "login") || + HasNestedGhCommand(arguments, "auth", "setup-git") || + HasNestedGhCommand(arguments, "gist", "clone") || + HasNestedGhCommand(arguments, "repo", "clone") || + HasNestedGhCommand(arguments, "pr", "checkout") || + (HasNestedGhCommand(arguments, "issue", "develop") && + HasGhOption(arguments, "--checkout", 'c')) || + (HasNestedGhCommand(arguments, "repo", "create") && + HasGhOption(arguments, "--clone", 'c')) || + (HasNestedGhCommand(arguments, "repo", "fork") && + HasGhOption(arguments, "--clone", 'c')); + } + + private static bool HasNestedGhCommand( + IReadOnlyList arguments, + string topLevel, + string nested) => + arguments[1] == topLevel && + arguments.Skip(2).TakeWhile(argument => argument != "--") + .Any(argument => argument == nested); + + private static bool HasGhProcessLaunchingOption(IReadOnlyList arguments) => + arguments.Skip(2).TakeWhile(argument => argument != "--") + .Any(argument => + argument is "--web" or "--browser" or "--editor" || + argument.StartsWith("--web=", StringComparison.Ordinal) || + argument.StartsWith("--browser=", StringComparison.Ordinal) || + argument.StartsWith("--editor=", StringComparison.Ordinal) || + HasShortGhOption(argument, 'w') || + HasShortGhOption(argument, 'e')); + + private static bool HasGhOption( + IReadOnlyList arguments, + string longOption, + char shortOption) => + arguments.Skip(2).TakeWhile(argument => argument != "--") + .Any(argument => + argument == longOption || + argument.StartsWith(longOption + "=", StringComparison.Ordinal) || + HasShortGhOption(argument, shortOption)); + + private static bool HasShortGhOption(string argument, char option) => + argument.Length > 1 && + argument[0] == '-' && + argument[1] != '-' && + argument.AsSpan(1).IndexOf(option) >= 0; + private static bool TryParseCommand( string command, out IReadOnlyList arguments, @@ -329,6 +394,7 @@ private static bool TryParseCommand( var current = new System.Text.StringBuilder(); var quote = '\0'; var escaping = false; + var argumentStarted = false; foreach (var character in command) { @@ -342,18 +408,21 @@ private static bool TryParseCommand( if (escaping) { current.Append(character); + argumentStarted = true; escaping = false; continue; } if (character == '\\' && quote != '\'') { + argumentStarted = true; escaping = true; continue; } if (character is '\'' or '"') { + argumentStarted = true; if (quote == '\0') quote = character; else if (quote == character) @@ -365,15 +434,17 @@ private static bool TryParseCommand( if (char.IsWhiteSpace(character) && quote == '\0') { - if (current.Length > 0) + if (argumentStarted) { parsed.Add(current.ToString()); current.Clear(); + argumentStarted = false; } continue; } current.Append(character); + argumentStarted = true; } if (escaping || quote != '\0') @@ -381,13 +452,26 @@ private static bool TryParseCommand( error = "Command rejected: GitHub credentials require balanced, literal arguments."; return false; } - if (current.Length > 0) + if (argumentStarted) parsed.Add(current.ToString()); arguments = parsed; return true; } + private static string FormatParsedCommand(IReadOnlyList arguments) => + string.Join(' ', arguments.Select(FormatParsedArgument)); + + private static string FormatParsedArgument(string argument) + { + if (argument.Length > 0 && argument.All(character => + char.IsLetterOrDigit(character) || + character is '-' or '_' or '.' or '/' or ':' or '=' or '+' or ',' or '@' or '%' or '#')) + return argument; + + return "'" + argument.Replace("'", "'\\''", StringComparison.Ordinal) + "'"; + } + private static bool IsDestructivePattern(string command, string[] patterns) { if (patterns.Length == 0) return false; diff --git a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs index a16c258b3..c04048a47 100644 --- a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs @@ -291,6 +291,132 @@ public async Task Controlled_run_command_requires_operator_approval_for_sensitiv executor.ExecuteCalls.Should().Be(0); } + [Theory] + [InlineData("gh 'secret' set DEPLOY_KEY --body value")] + [InlineData("gh \"secret\" set DEPLOY_KEY --body value")] + [InlineData("gh 'auth' token")] + [InlineData("gh \"auth\" token")] + public async Task Controlled_run_command_requires_operator_approval_for_quoted_sensitive_gh_commands( + string command) + { + var executor = new CountingExecutor(); + using var tracker = new ShellExecutionTracker(); + var context = BuildContext( + executor, + tracker, + repositoryAccessToken: "repository-access-token", + destructivePatterns: [.. SandboxPolicy.Default(_root).DestructiveCommandPatterns], + rejectDestructiveCommands: false); + var tool = CopilotAIAgent.BuildSessionConfigTools( + context, + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var result = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = command })); + + result?.ToString().Should().Contain("requires operator approval"); + executor.ExecuteCalls.Should().Be(0); + } + + [Fact] + public async Task Controlled_run_command_uses_one_approval_identity_for_equivalent_quoted_gh_arguments() + { + var observedHashes = new List(); + var executor = new CountingExecutor(); + using var tracker = new ShellExecutionTracker(); + var context = BuildContext( + executor, + tracker, + repositoryAccessToken: "repository-access-token", + destructivePatterns: [.. SandboxPolicy.Default(_root).DestructiveCommandPatterns], + rejectDestructiveCommands: false, + isCommandApproved: hash => + { + observedHashes.Add(hash); + return false; + }); + var tool = CopilotAIAgent.BuildSessionConfigTools( + context, + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = "gh secret set DEPLOY_KEY --body value" })); + await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = "gh \"secret\" set DEPLOY_KEY --body value" })); + + observedHashes.Should().HaveCount(2); + observedHashes.Distinct().Should().ContainSingle( + "equivalent quoted spellings must resolve to the same direct-execution argv identity"); + executor.ExecuteCalls.Should().Be(0); + } + + [Fact] + public async Task Controlled_run_command_preserves_safe_quoted_gh_arguments_for_direct_execution() + { + SandboxCommand? observed = null; + var executor = new CapturingExecutor(command => observed = command); + using var tracker = new ShellExecutionTracker(); + var tool = CopilotAIAgent.BuildSessionConfigTools( + BuildContext(executor, tracker, repositoryAccessToken: "repository-access-token"), + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var result = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary + { + ["command"] = "gh \"repo\" view \"sabbour/agentweaver\"", + })); + + result?.ToString().Should().Contain("exit_code: 0"); + observed!.DirectExecution!.Arguments.Should().Equal("repo", "view", "sabbour/agentweaver"); + } + + [Theory] + [InlineData("gh pr create --editor")] + [InlineData("gh pr create -e")] + [InlineData("gh pr create -ew")] + [InlineData("gh pr create -we")] + [InlineData("gh pr view 1 --web")] + [InlineData("gh pr view 1 --web=true")] + [InlineData("gh pr view 1 -w")] + [InlineData("gh pr view 1 -we")] + [InlineData("gh pr view 1 -ew")] + [InlineData("gh repo view --browser")] + [InlineData("gh gist clone deadbeef")] + [InlineData("gh issue develop 1 --checkout")] + [InlineData("gh issue develop 1 -c")] + [InlineData("gh repo create example --clone")] + [InlineData("gh repo create example -c")] + [InlineData("gh repo fork example --clone")] + [InlineData("gh repo fork example -c")] + [InlineData("gh auth login")] + [InlineData("gh auth setup-git")] + [InlineData("gh codespace list")] + public async Task Controlled_run_command_does_not_execute_approved_gh_child_process_paths( + string command) + { + const string sentinel = "sentinel-repository-token"; + SandboxCommand? observed = null; + var executor = new CapturingExecutor(candidate => observed = candidate); + using var tracker = new ShellExecutionTracker(); + var context = BuildContext( + executor, + tracker, + repositoryAccessToken: sentinel, + requireApprovalForAllShell: true, + isCommandApproved: _ => true); + var tool = CopilotAIAgent.BuildSessionConfigTools( + context, + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var result = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = command })); + + result?.ToString().Should().Contain("start another executable"); + result?.ToString().Should().NotContain(sentinel); + observed.Should().BeNull( + "an editor, browser, clone, checkout, or nested gh child must never receive the repository token"); + } + [Fact] public async Task Controlled_run_command_does_not_leak_the_sentinel_to_a_repository_configured_external_diff() { @@ -600,7 +726,9 @@ private SandboxToolContext BuildContext( string? scratchDirectory = null, string? repositoryAccessToken = null, string[]? destructivePatterns = null, - bool rejectDestructiveCommands = true) => + bool rejectDestructiveCommands = true, + bool requireApprovalForAllShell = false, + Func? isCommandApproved = null) => new( AgentId: "agent", WorkingDirectory: workspace ?? _root, @@ -614,11 +742,13 @@ private SandboxToolContext BuildContext( DestructiveCommandPatterns = destructivePatterns ?? ["rm -rf"], RejectBackgroundCommands = true, RejectDestructiveCommands = rejectDestructiveCommands, + RequireApprovalForAllShell = requireApprovalForAllShell, MaximumTimeoutMs = 600_000, RepositoryAccessToken = repositoryAccessToken, }, Logger: NullLogger.Instance, ShellExecutionTracker: tracker, + IsCommandApproved: isCommandApproved, ScratchDirectory: scratchDirectory); private static async Task RunGitAsync(string workingDirectory, params string[] arguments) From 239f6c43ea16f2307b48152bcc95bc920122699d Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 21:49:28 -0700 Subject: [PATCH 07/12] fix: allow only direct credentialed gh commands Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- docs/deep-dive/sandboxed-execution.md | 7 + .../Tools/RunCommandTool.cs | 159 +++++++++++------- .../AssemblyBuildTestShellGuardTests.cs | 38 ++++- 3 files changed, 141 insertions(+), 63 deletions(-) diff --git a/docs/deep-dive/sandboxed-execution.md b/docs/deep-dive/sandboxed-execution.md index ec2e54b8c..4692a09c5 100644 --- a/docs/deep-dive/sandboxed-execution.md +++ b/docs/deep-dive/sandboxed-execution.md @@ -96,6 +96,13 @@ merge drivers, hooks, aliases, helpers, or remote helpers. The direct Git proces credential helpers, filesystem monitors, and recursive submodules. Its GitHub authorization header is scoped to that process; it is never supplied to a child process. +Credential-bearing `gh` invocations must also match a narrow parsed allowlist. It permits direct +`gh api` and `gh status` calls; explicitly named repository list, view, and fork calls; and +explicitly repository-scoped issue, pull-request, and workflow forms that do not launch another +program. `gh issue develop` is limited to its `--list` form. Repository forks that clone or alter +remotes, pull-request creation and checkout, branch-deleting close or merge commands, and +browser/editor forms are rejected even after operator approval, so they never receive `GH_TOKEN`. + The approval policy gates `git push`, remote changes, `gh pr` changes, `gh repo` changes, `gh api`, `gh secret set`, and `gh auth` commands (including `gh auth token`). The API does not inspect or proxy these commands. diff --git a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs index 29121e40a..ac1ae0dd5 100644 --- a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs +++ b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs @@ -219,13 +219,6 @@ private static Dictionary BuildCommandEnvironment( private const string CredentialSafeGitCommand = "status"; - private static readonly HashSet BuiltInGhCommands = new(StringComparer.Ordinal) - { - "api", "attestation", "auth", "cache", "codespace", "completion", "config", - "gist", "gpg-key", "issue", "label", "org", "pr", "project", "release", "repo", - "ruleset", "search", "secret", "ssh-key", "status", "variable", "workflow", - }; - private static bool TryCreateRepositoryCredentialCommand( IReadOnlyList? arguments, string? accessToken, @@ -316,72 +309,118 @@ private static bool TryValidateGhArguments( out string? error) { error = null; - if (arguments.Count < 2 || !BuiltInGhCommands.Contains(arguments[1])) + if (!IsDirectGhCommand(arguments)) { - error = "Command rejected: repository credentials require a built-in gh command."; - return false; - } - - if (GhArgumentsCanStartAnotherExecutable(arguments)) - { - error = "Command rejected: gh commands that start another executable are not allowed with repository credentials."; + error = "Command rejected: repository credentials only allow parsed direct gh command forms."; return false; } return true; } - private static bool GhArgumentsCanStartAnotherExecutable(IReadOnlyList arguments) + private static bool IsDirectGhCommand(IReadOnlyList arguments) => + // `gh api` cannot expand repository placeholders here because TryParseCommand rejects + // braces before this allowlist is evaluated. + IsGhCommand(arguments, "api") || + IsGhCommand(arguments, "status") || + HasExactSingleGhArgument(arguments, "repo", "list", IsGhIdentifier) || + HasExactSingleGhArgument(arguments, "repo", "view", IsExplicitRepository) || + HasExactSingleGhArgument(arguments, "repo", "fork", IsExplicitRepository) || + HasGhRepositoryOption(arguments, ["issue", "list"], positionalArgumentCount: 0) || + HasGhRepositoryOption(arguments, ["issue", "view"], positionalArgumentCount: 1) || + HasGhRepositoryOption( + arguments, + ["issue", "develop"], + positionalArgumentCount: 1, + requiredOption: "--list") || + HasGhRepositoryOption(arguments, ["pr", "list"], positionalArgumentCount: 0) || + HasGhRepositoryOption(arguments, ["pr", "view"], positionalArgumentCount: 1) || + HasGhRepositoryOption(arguments, ["pr", "close"], positionalArgumentCount: 1) || + HasGhRepositoryOption(arguments, ["pr", "merge"], positionalArgumentCount: 1) || + HasGhRepositoryOption(arguments, ["workflow", "list"], positionalArgumentCount: 0) || + HasGhRepositoryOption(arguments, ["workflow", "view"], positionalArgumentCount: 1) || + HasGhRepositoryOption(arguments, ["workflow", "run"], positionalArgumentCount: 1); + + private static bool HasExactSingleGhArgument( + IReadOnlyList arguments, + string topLevelCommand, + string subcommand, + Func isAllowedArgument) => + IsGhCommand(arguments, topLevelCommand, subcommand) && + arguments.Count == 4 && + isAllowedArgument(arguments[3]); + + private static bool HasGhRepositoryOption( + IReadOnlyList arguments, + IReadOnlyList command, + int positionalArgumentCount, + string? requiredOption = null) { - if (arguments[1] == "codespace" || HasGhProcessLaunchingOption(arguments)) - return true; + if (!IsGhCommand(arguments, command)) + return false; + + var positionalArguments = new List(positionalArgumentCount); + var hasRepository = false; + var hasRequiredOption = requiredOption is null; + for (var index = command.Count + 1; index < arguments.Count; index++) + { + var argument = arguments[index]; + if (argument is "--repo" or "-R") + { + if (hasRepository || ++index >= arguments.Count || !IsExplicitRepository(arguments[index])) + return false; - return HasNestedGhCommand(arguments, "auth", "login") || - HasNestedGhCommand(arguments, "auth", "setup-git") || - HasNestedGhCommand(arguments, "gist", "clone") || - HasNestedGhCommand(arguments, "repo", "clone") || - HasNestedGhCommand(arguments, "pr", "checkout") || - (HasNestedGhCommand(arguments, "issue", "develop") && - HasGhOption(arguments, "--checkout", 'c')) || - (HasNestedGhCommand(arguments, "repo", "create") && - HasGhOption(arguments, "--clone", 'c')) || - (HasNestedGhCommand(arguments, "repo", "fork") && - HasGhOption(arguments, "--clone", 'c')); + hasRepository = true; + continue; + } + + if (argument.StartsWith("--repo=", StringComparison.Ordinal)) + { + if (hasRepository || !IsExplicitRepository(argument["--repo=".Length..])) + return false; + + hasRepository = true; + continue; + } + + if (argument == requiredOption) + { + if (hasRequiredOption) + return false; + + hasRequiredOption = true; + continue; + } + + if (!IsGhIdentifier(argument)) + return false; + + positionalArguments.Add(argument); + } + + return hasRepository && + hasRequiredOption && + positionalArguments.Count == positionalArgumentCount; } - private static bool HasNestedGhCommand( + private static bool IsGhCommand( IReadOnlyList arguments, - string topLevel, - string nested) => - arguments[1] == topLevel && - arguments.Skip(2).TakeWhile(argument => argument != "--") - .Any(argument => argument == nested); - - private static bool HasGhProcessLaunchingOption(IReadOnlyList arguments) => - arguments.Skip(2).TakeWhile(argument => argument != "--") - .Any(argument => - argument is "--web" or "--browser" or "--editor" || - argument.StartsWith("--web=", StringComparison.Ordinal) || - argument.StartsWith("--browser=", StringComparison.Ordinal) || - argument.StartsWith("--editor=", StringComparison.Ordinal) || - HasShortGhOption(argument, 'w') || - HasShortGhOption(argument, 'e')); - - private static bool HasGhOption( + IReadOnlyList command) => + arguments.Count > command.Count && + arguments[0] == "gh" && + command.Select((word, index) => arguments[index + 1] == word).All(matches => matches); + + private static bool IsGhCommand( IReadOnlyList arguments, - string longOption, - char shortOption) => - arguments.Skip(2).TakeWhile(argument => argument != "--") - .Any(argument => - argument == longOption || - argument.StartsWith(longOption + "=", StringComparison.Ordinal) || - HasShortGhOption(argument, shortOption)); - - private static bool HasShortGhOption(string argument, char option) => - argument.Length > 1 && - argument[0] == '-' && - argument[1] != '-' && - argument.AsSpan(1).IndexOf(option) >= 0; + params string[] command) => + IsGhCommand(arguments, (IReadOnlyList)command); + + private static bool IsExplicitRepository(string argument) => + IsGhIdentifier(argument) && argument.Contains("/", StringComparison.Ordinal); + + private static bool IsGhIdentifier(string argument) => + !string.IsNullOrWhiteSpace(argument) && + !argument.StartsWith("-", StringComparison.Ordinal); private static bool TryParseCommand( string command, diff --git a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs index c04048a47..4cd322780 100644 --- a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs @@ -132,7 +132,21 @@ public async Task Controlled_run_command_rejects_backgrounding_and_destructive_c [Theory] [InlineData("git status")] - [InlineData("gh repo view")] + [InlineData("gh api user")] + [InlineData("gh status")] + [InlineData("gh repo list sabbour")] + [InlineData("gh repo view sabbour/agentweaver")] + [InlineData("gh repo fork sabbour/agentweaver")] + [InlineData("gh issue list --repo sabbour/agentweaver")] + [InlineData("gh issue view 947 --repo sabbour/agentweaver")] + [InlineData("gh issue develop 947 --list --repo sabbour/agentweaver")] + [InlineData("gh pr list --repo sabbour/agentweaver")] + [InlineData("gh pr view 968 --repo sabbour/agentweaver")] + [InlineData("gh pr close 968 --repo sabbour/agentweaver")] + [InlineData("gh pr merge 968 --repo sabbour/agentweaver")] + [InlineData("gh workflow list --repo sabbour/agentweaver")] + [InlineData("gh workflow view ci.yml --repo sabbour/agentweaver")] + [InlineData("gh workflow run ci.yml --repo sabbour/agentweaver")] public async Task Controlled_run_command_uses_direct_execution_for_repository_credential_commands(string command) { SandboxCommand? observed = null; @@ -373,6 +387,7 @@ public async Task Controlled_run_command_preserves_safe_quoted_gh_arguments_for_ [Theory] [InlineData("gh pr create --editor")] [InlineData("gh pr create -e")] + [InlineData("gh pr create --title test --body test")] [InlineData("gh pr create -ew")] [InlineData("gh pr create -we")] [InlineData("gh pr view 1 --web")] @@ -381,13 +396,23 @@ public async Task Controlled_run_command_preserves_safe_quoted_gh_arguments_for_ [InlineData("gh pr view 1 -we")] [InlineData("gh pr view 1 -ew")] [InlineData("gh repo view --browser")] + [InlineData("gh repo view sabbour/agentweaver --web")] [InlineData("gh gist clone deadbeef")] + [InlineData("gh issue develop 1")] [InlineData("gh issue develop 1 --checkout")] [InlineData("gh issue develop 1 -c")] + [InlineData("gh issue list --web")] [InlineData("gh repo create example --clone")] [InlineData("gh repo create example -c")] + [InlineData("gh repo fork example --remote")] [InlineData("gh repo fork example --clone")] [InlineData("gh repo fork example -c")] + [InlineData("gh pr checkout 1")] + [InlineData("gh pr close 1 --delete-branch")] + [InlineData("gh pr close 1 -d")] + [InlineData("gh pr merge 1 --delete-branch")] + [InlineData("gh pr merge 1 -d")] + [InlineData("gh workflow view build.yml --web")] [InlineData("gh auth login")] [InlineData("gh auth setup-git")] [InlineData("gh codespace list")] @@ -396,6 +421,7 @@ public async Task Controlled_run_command_does_not_execute_approved_gh_child_proc { const string sentinel = "sentinel-repository-token"; SandboxCommand? observed = null; + var approvalChecks = 0; var executor = new CapturingExecutor(candidate => observed = candidate); using var tracker = new ShellExecutionTracker(); var context = BuildContext( @@ -403,7 +429,11 @@ public async Task Controlled_run_command_does_not_execute_approved_gh_child_proc tracker, repositoryAccessToken: sentinel, requireApprovalForAllShell: true, - isCommandApproved: _ => true); + isCommandApproved: _ => + { + approvalChecks++; + return true; + }); var tool = CopilotAIAgent.BuildSessionConfigTools( context, includeControlledRunCommand: true).Single(t => t.Name == "run_command"); @@ -411,8 +441,10 @@ public async Task Controlled_run_command_does_not_execute_approved_gh_child_proc var result = await tool.InvokeAsync(new AIFunctionArguments( new Dictionary { ["command"] = command })); - result?.ToString().Should().Contain("start another executable"); + result?.ToString().Should().Contain("parsed direct gh command forms"); result?.ToString().Should().NotContain(sentinel); + approvalChecks.Should().Be(1, + "an approval must not make a non-allowlisted command eligible for the repository token"); observed.Should().BeNull( "an editor, browser, clone, checkout, or nested gh child must never receive the repository token"); } From 760268979483796af96efdfa0ed1fd5a276f381a Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 21:54:18 -0700 Subject: [PATCH 08/12] fix: block persisted gh editor credential paths Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- docs/deep-dive/sandboxed-execution.md | 3 + .../Tools/RunCommandTool.cs | 2 + .../AssemblyBuildTestShellGuardTests.cs | 114 ++++++++++++++++++ 3 files changed, 119 insertions(+) diff --git a/docs/deep-dive/sandboxed-execution.md b/docs/deep-dive/sandboxed-execution.md index 4692a09c5..2aaa2a49b 100644 --- a/docs/deep-dive/sandboxed-execution.md +++ b/docs/deep-dive/sandboxed-execution.md @@ -102,6 +102,9 @@ explicitly repository-scoped issue, pull-request, and workflow forms that do not program. `gh issue develop` is limited to its `--list` form. Repository forks that clone or alter remotes, pull-request creation and checkout, branch-deleting close or merge commands, and browser/editor forms are rejected even after operator approval, so they never receive `GH_TOKEN`. +`gh config set` is not allowlisted: editor and pager settings persist and can make a later command +start an external program. Likewise, `gh gist edit` and every other editor-facing gist form remain +outside the allowlist. The approval policy gates `git push`, remote changes, `gh pr` changes, `gh repo` changes, `gh api`, `gh secret set`, and `gh auth` commands (including `gh auth token`). The API does not inspect or diff --git a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs index ac1ae0dd5..c5d361166 100644 --- a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs +++ b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs @@ -321,6 +321,8 @@ private static bool TryValidateGhArguments( private static bool IsDirectGhCommand(IReadOnlyList arguments) => // `gh api` cannot expand repository placeholders here because TryParseCommand rejects // braces before this allowlist is evaluated. + // `gh config` and `gh gist` are intentionally absent: persisted settings can configure + // helpers that a later gist edit command would launch with the inherited token. IsGhCommand(arguments, "api") || IsGhCommand(arguments, "status") || HasExactSingleGhArgument(arguments, "repo", "list", IsGhIdentifier) || diff --git a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs index 4cd322780..ec110e54f 100644 --- a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs @@ -398,6 +398,9 @@ public async Task Controlled_run_command_preserves_safe_quoted_gh_arguments_for_ [InlineData("gh repo view --browser")] [InlineData("gh repo view sabbour/agentweaver --web")] [InlineData("gh gist clone deadbeef")] + [InlineData("gh gist edit deadbeef")] + [InlineData("gh config set editor credential-observer.sh")] + [InlineData("gh config set pager credential-observer.sh")] [InlineData("gh issue develop 1")] [InlineData("gh issue develop 1 --checkout")] [InlineData("gh issue develop 1 -c")] @@ -449,6 +452,49 @@ public async Task Controlled_run_command_does_not_execute_approved_gh_child_proc "an editor, browser, clone, checkout, or nested gh child must never receive the repository token"); } + [Fact] + public async Task Controlled_run_command_blocks_a_persisted_gh_editor_before_it_receives_the_sentinel() + { + const string sentinel = "sentinel-repository-token"; + var observerOutput = Path.Combine(_root, "gh-editor-token.txt"); + var observerExecutable = Path.Combine(_root, "credential-observer.sh"); + WriteEnvironmentObserver(observerExecutable, observerOutput); + var executor = new PersistedGhEditorExecutor(); + var approvalChecks = 0; + using var tracker = new ShellExecutionTracker(); + var tool = CopilotAIAgent.BuildSessionConfigTools( + BuildContext( + executor, + tracker, + repositoryAccessToken: sentinel, + requireApprovalForAllShell: true, + isCommandApproved: _ => + { + approvalChecks++; + return true; + }), + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var configSet = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary + { + ["command"] = "gh config set editor credential-observer.sh", + })); + var gistEdit = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary + { + ["command"] = "gh gist edit deadbeef", + })); + + configSet?.ToString().Should().Contain("parsed direct gh command forms"); + gistEdit?.ToString().Should().Contain("parsed direct gh command forms"); + approvalChecks.Should().Be(2, + "approval must not make persisted editor configuration or gist editing credential-eligible"); + executor.ExecuteCalls.Should().Be(0); + File.Exists(observerOutput).Should().BeFalse( + "the workspace editor script must never receive the credential-bearing gh process environment"); + } + [Fact] public async Task Controlled_run_command_does_not_leak_the_sentinel_to_a_repository_configured_external_diff() { @@ -882,6 +928,74 @@ public async IAsyncEnumerable StreamAsync( } } + private sealed class PersistedGhEditorExecutor : ISandboxExecutor + { + private string? _editor; + + public int ExecuteCalls { get; private set; } + public bool IsRealIsolation => false; + public string BackendName => "direct"; + public string SelectionReason => "test"; + public bool HasNetworkWarning => false; + public string? NetworkWarningMessage => null; + + public async Task ExecuteAsync( + SandboxCommand command, + CancellationToken ct = default) + { + ExecuteCalls++; + var directExecution = command.DirectExecution + ?? throw new InvalidOperationException("The credential test expects direct gh execution."); + + if (directExecution.Executable == "gh" && + directExecution.Arguments.Count == 4 && + directExecution.Arguments[0] == "config" && + directExecution.Arguments[1] == "set" && + directExecution.Arguments[2] == "editor") + { + _editor = directExecution.Arguments[3]; + } + else if (directExecution.Executable == "gh" && + directExecution.Arguments.Count >= 2 && + directExecution.Arguments[0] == "gist" && + directExecution.Arguments[1] == "edit" && + _editor is not null) + { + var editor = new ProcessStartInfo + { + FileName = "sh", + WorkingDirectory = command.WorkingDirectory, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + editor.ArgumentList.Add(_editor); + if (directExecution.Environment is { } environment) + { + foreach (var pair in environment) + editor.Environment[pair.Key] = pair.Value; + } + + using var process = Process.Start(editor) + ?? throw new InvalidOperationException("Could not start configured editor."); + await process.WaitForExitAsync(ct); + if (process.ExitCode != 0) + throw new InvalidOperationException(await process.StandardError.ReadToEndAsync(ct)); + } + + return new SandboxExecResult(0, "ok", "", TimedOut: false, OutputTruncated: false); + } + + public async IAsyncEnumerable StreamAsync( + SandboxCommand command, + [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken ct = default) + { + var result = await ExecuteAsync(command, ct); + yield return new SandboxOutputChunk(SandboxOutputStream.Stdout, result.Stdout); + } + } + private sealed class CountingExecutor(bool blockFirstCall = false) : ISandboxExecutor { private int _active; From 0ac169b36445f6e0a50d0ff704a9f8ed8565f56f Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 22:11:45 -0700 Subject: [PATCH 09/12] fix: repair repository credential lifecycle Refs #947 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- .../RunRepositoryCredentialRegistry.cs | 91 +++++++++++++++++-- .../Tools/RunCommandTool.cs | 36 ++++++-- .../AssemblyBuildTestShellGuardTests.cs | 34 ++++++- .../RunRepositoryCredentialRegistryTests.cs | 75 +++++++++++++++ 4 files changed, 214 insertions(+), 22 deletions(-) diff --git a/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs index 97d007ff7..797198509 100644 --- a/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs +++ b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs @@ -21,7 +21,17 @@ public sealed class RunRepositoryCredentialRegistry private readonly ConcurrentDictionary _entries = new(StringComparer.Ordinal); private readonly ConcurrentDictionary _retainedRevocations = new(StringComparer.Ordinal); - private readonly ConcurrentDictionary _mintLocks = new(StringComparer.Ordinal); + private readonly object _mintLockGate = new(); + private readonly Dictionary _mintLocks = new(StringComparer.Ordinal); + + internal int ActiveRunLockCount + { + get + { + lock (_mintLockGate) + return _mintLocks.Count; + } + } public RunRepositoryCredentialRegistry(IServiceScopeFactory scopeFactory) : this(new RunRepositoryCredentialMinter(scopeFactory)) @@ -38,8 +48,8 @@ internal RunRepositoryCredentialRegistry( public async Task MintAsync(string runId, CancellationToken ct = default) { - var mintLock = _mintLocks.GetOrAdd(runId, static _ => new SemaphoreSlim(1, 1)); - await mintLock.WaitAsync(ct).ConfigureAwait(false); + using var mintLockLease = AcquireMintLock(runId); + await mintLockLease.Semaphore.WaitAsync(ct).ConfigureAwait(false); try { var now = _timeProvider.GetUtcNow(); @@ -67,7 +77,7 @@ internal RunRepositoryCredentialRegistry( } finally { - mintLock.Release(); + mintLockLease.Semaphore.Release(); } } @@ -76,8 +86,8 @@ public async Task RevokeAsync(string? runId, CancellationToken ct = default) if (string.IsNullOrWhiteSpace(runId)) return; - var mintLock = _mintLocks.GetOrAdd(runId, static _ => new SemaphoreSlim(1, 1)); - await mintLock.WaitAsync(ct).ConfigureAwait(false); + using var mintLockLease = AcquireMintLock(runId); + await mintLockLease.Semaphore.WaitAsync(ct).ConfigureAwait(false); try { var now = _timeProvider.GetUtcNow(); @@ -103,7 +113,7 @@ public async Task RevokeAsync(string? runId, CancellationToken ct = default) } finally { - mintLock.Release(); + mintLockLease.Semaphore.Release(); } } @@ -120,8 +130,8 @@ internal async Task> RetryFa { ct.ThrowIfCancellationRequested(); - var mintLock = _mintLocks.GetOrAdd(runId, static _ => new SemaphoreSlim(1, 1)); - await mintLock.WaitAsync(ct).ConfigureAwait(false); + using var mintLockLease = AcquireMintLock(runId); + await mintLockLease.Semaphore.WaitAsync(ct).ConfigureAwait(false); try { if (!_retainedRevocations.TryGetValue(runId, out var retained)) @@ -156,13 +166,50 @@ internal async Task> RetryFa } finally { - mintLock.Release(); + mintLockLease.Semaphore.Release(); } } return failures; } + private RunCredentialLockLease AcquireMintLock(string runId) + { + lock (_mintLockGate) + { + if (!_mintLocks.TryGetValue(runId, out var mintLock)) + { + mintLock = new RunCredentialLock(); + _mintLocks.Add(runId, mintLock); + } + + mintLock.ActiveOperations++; + return new RunCredentialLockLease(this, runId, mintLock); + } + } + + private void ReleaseMintLock(string runId, RunCredentialLock mintLock) + { + lock (_mintLockGate) + { + if (mintLock.ActiveOperations <= 0) + throw new InvalidOperationException("Run credential lock lease was released more than once."); + + mintLock.ActiveOperations--; + if (mintLock.ActiveOperations != 0 || + _entries.ContainsKey(runId) || + _retainedRevocations.ContainsKey(runId) || + !_mintLocks.TryGetValue(runId, out var current) || + !ReferenceEquals(current, mintLock)) + { + return; + } + + _mintLocks.Remove(runId); + mintLock.Dispose(); + } + } + private async Task RevokeAndRemoveAsync( string runId, string accessToken, @@ -210,6 +257,30 @@ private sealed record RetainedRevocation( DateTimeOffset ExpiresAt, int FailureCount, DateTimeOffset NextAttemptAt); + + private sealed class RunCredentialLock : IDisposable + { + public SemaphoreSlim Semaphore { get; } = new(1, 1); + public int ActiveOperations { get; set; } + + public void Dispose() => Semaphore.Dispose(); + } + + private sealed class RunCredentialLockLease( + RunRepositoryCredentialRegistry registry, + string runId, + RunCredentialLock mintLock) : IDisposable + { + private RunRepositoryCredentialRegistry? _registry = registry; + + public SemaphoreSlim Semaphore => mintLock.Semaphore; + + public void Dispose() + { + var registry = Interlocked.Exchange(ref _registry, null); + registry?.ReleaseMintLock(runId, mintLock); + } + } } internal sealed record FailedRepositoryCredentialRevocation(string RunId, Exception Exception); diff --git a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs index c5d361166..e88dc3f9c 100644 --- a/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs +++ b/packages/Agentweaver.AgentTools/Tools/RunCommandTool.cs @@ -22,19 +22,16 @@ public AIFunction CreateFunction(SandboxToolContext ctx) => IReadOnlyList? credentialArguments = null; var approvalCommand = command; var commandHash = ComputeCommandHash(command); - if (!string.IsNullOrWhiteSpace(ctx.Options.RepositoryAccessToken)) + if (!string.IsNullOrWhiteSpace(ctx.Options.RepositoryAccessToken) && + BeginsCredentialBearingCommand(command)) { if (!TryParseCommand(command, out credentialArguments, out var credentialParseError)) return credentialParseError!; - if (credentialArguments.Count > 0 && - (credentialArguments[0] == "git" || credentialArguments[0] == "gh")) - { - // Approval must describe and identify the exact argv that receives the - // credential, rather than the shell spelling the model originally sent. - approvalCommand = FormatParsedCommand(credentialArguments); - commandHash = ComputeCommandHash(credentialArguments); - } + // Approval must describe and identify the exact argv that receives the + // credential, rather than the shell spelling the model originally sent. + approvalCommand = FormatParsedCommand(credentialArguments); + commandHash = ComputeCommandHash(credentialArguments); } var destructive = IsDestructivePattern(approvalCommand, ctx.Options.DestructiveCommandPatterns); @@ -230,8 +227,10 @@ private static bool TryCreateRepositoryCredentialCommand( if (string.IsNullOrWhiteSpace(accessToken)) return true; + // Commands that do not begin with git or gh retain the ordinary sandbox shell path. + // They receive no repository credential data. if (arguments is null) - throw new InvalidOperationException("Credential-bearing command arguments must be parsed before execution."); + return true; if (arguments.Count == 0 || (arguments[0] != "git" && arguments[0] != "gh")) return true; @@ -424,6 +423,23 @@ private static bool IsGhIdentifier(string argument) => !string.IsNullOrWhiteSpace(argument) && !argument.StartsWith("-", StringComparison.Ordinal); + private static bool BeginsCredentialBearingCommand(string command) => + HasCredentialCommandPrefix(command.TrimStart(), "git") || + HasCredentialCommandPrefix(command.TrimStart(), "gh"); + + private static bool HasCredentialCommandPrefix(string command, string executable) + { + if (!command.StartsWith(executable, StringComparison.Ordinal)) + return false; + if (command.Length == executable.Length) + return true; + + var next = command[executable.Length]; + return char.IsWhiteSpace(next) || + next is '\0' or ';' or '|' or '&' or '`' or '$' or '<' or '>' or '(' or ')' or + '{' or '}' or '[' or ']' or '*' or '?' or '!' or '~'; + } + private static bool TryParseCommand( string command, out IReadOnlyList arguments, diff --git a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs index ec110e54f..f37845ac1 100644 --- a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs @@ -202,6 +202,35 @@ await tool.InvokeAsync(new AIFunctionArguments( pair => pair.Value == "repository-access-token"); } + [Theory] + [InlineData("npm test && npm run lint")] + [InlineData("npm test -- tests/*.cs")] + [InlineData("npm test > test-output.txt")] + public async Task Controlled_run_command_keeps_normal_shell_syntax_uncredentialed( + string command) + { + const string sentinel = "repository-access-token"; + SandboxCommand? observed = null; + var executor = new CapturingExecutor(candidate => observed = candidate); + using var tracker = new ShellExecutionTracker(); + var tool = CopilotAIAgent.BuildSessionConfigTools( + BuildContext(executor, tracker, repositoryAccessToken: sentinel), + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var result = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = command })); + + result?.ToString().Should().Contain("exit_code: 0"); + observed.Should().NotBeNull(); + observed!.CommandLine.Should().Be(command); + observed.DirectExecution.Should().BeNull( + "only direct, allowlisted git or gh commands may receive repository credentials"); + observed.Environment.Should().NotContainKey("GH_TOKEN") + .And.NotContainKey("GITHUB_TOKEN") + .And.NotContainKey("GIT_CONFIG_PARAMETERS"); + observed.Environment.Should().NotContain(pair => pair.Value == sentinel); + } + [Fact] public async Task Controlled_run_command_redacts_repository_credential_from_command_output() { @@ -219,10 +248,11 @@ public async Task Controlled_run_command_redacts_repository_credential_from_comm [Theory] [InlineData("git status; whoami")] - [InlineData("gh repo view > output.txt")] + [InlineData("git status && npm test")] + [InlineData("gh api /user > output.txt")] [InlineData("git $(echo status)")] [InlineData("gh repo view\r\nwhoami")] - public async Task Controlled_run_command_rejects_compound_credentialed_commands(string command) + public async Task Controlled_run_command_rejects_shell_syntax_for_credentialed_commands(string command) { var executor = new CountingExecutor(); using var tracker = new ShellExecutionTracker(); diff --git a/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs b/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs index f18567df2..28c908d0c 100644 --- a/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialRegistryTests.cs @@ -23,6 +23,8 @@ public async Task Revoke_RetainsCredentialAfterThrownFailure_ThenRetriesAndClean (await registry.MintAsync("run-credential-retry")).Should().Be("registry-sentinel-token"); var first = () => registry.RevokeAsync("run-credential-retry"); await first.Should().ThrowAsync(); + registry.ActiveRunLockCount.Should().Be(1, + "retained revocation state must keep its per-run lock available for a retry"); await registry.RevokeAsync("run-credential-retry"); (await registry.RetryFailedRevocationsAsync()).Should().BeEmpty( @@ -36,6 +38,8 @@ public async Task Revoke_RetainsCredentialAfterThrownFailure_ThenRetriesAndClean minter.RevokedTokens.Should().HaveCount(2, "a failed revoke must retain its retry state, while a later automatic success removes it"); minter.RevokedTokens.Should().OnlyContain(token => token == "registry-sentinel-token"); + registry.ActiveRunLockCount.Should().Be(0, + "a successful terminal retry must remove and dispose the run's unused lock"); } [Fact] @@ -64,6 +68,50 @@ public async Task Retry_DropsRetainedCredentialOnlyAfterActualExpiry() "an expired credential must not be sent for another provider revocation attempt"); } + [Fact] + public async Task Revoke_TerminalCleanup_RemovesRunLock() + { + var now = new DateTimeOffset(2026, 8, 27, 20, 0, 0, TimeSpan.Zero); + var minter = new StubCredentialMinter + { + Credential = new RepositoryCredential("terminal-cleanup-token", now.AddMinutes(5)), + }; + var registry = new RunRepositoryCredentialRegistry(minter, new MutableTimeProvider(now)); + + (await registry.MintAsync("run-terminal-cleanup")).Should().Be("terminal-cleanup-token"); + registry.ActiveRunLockCount.Should().Be(1, + "an active credential must retain the lock that serializes its terminal revocation"); + + await registry.RevokeAsync("run-terminal-cleanup"); + + minter.RevokedTokens.Should().ContainSingle().Which.Should().Be("terminal-cleanup-token"); + registry.ActiveRunLockCount.Should().Be(0, + "no credential, retained revocation, or operation remains after terminal cleanup"); + } + + [Fact] + public async Task ConcurrentMintAndRevoke_KeepTheRunLockUntilBothOperationsComplete() + { + var now = new DateTimeOffset(2026, 8, 27, 20, 0, 0, TimeSpan.Zero); + var minter = new BlockingCredentialMinter( + new RepositoryCredential("concurrent-cleanup-token", now.AddMinutes(5))); + var registry = new RunRepositoryCredentialRegistry(minter, new MutableTimeProvider(now)); + + var mint = registry.MintAsync("run-concurrent-cleanup"); + await minter.MintStarted; + var revoke = registry.RevokeAsync("run-concurrent-cleanup"); + registry.ActiveRunLockCount.Should().Be(1, + "the queued revocation holds a lease before the mint operation can release its lock"); + + minter.AllowMint(); + (await mint).Should().Be("concurrent-cleanup-token"); + await revoke; + + minter.RevokedTokens.Should().ContainSingle().Which.Should().Be("concurrent-cleanup-token"); + registry.ActiveRunLockCount.Should().Be(0, + "the lock is disposed only after the queued terminal revocation completes"); + } + private sealed class StubCredentialMinter : IRunRepositoryCredentialMinter { public RepositoryCredential? Credential { get; init; } @@ -82,6 +130,33 @@ public Task RevokeAsync(string accessToken, CancellationToken ct) } } + private sealed class BlockingCredentialMinter(RepositoryCredential credential) + : IRunRepositoryCredentialMinter + { + private readonly TaskCompletionSource _mintStarted = + new(TaskCreationOptions.RunContinuationsAsynchronously); + private readonly TaskCompletionSource _allowMint = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public List RevokedTokens { get; } = []; + public Task MintStarted => _mintStarted.Task; + + public async Task MintAsync(string runId, CancellationToken ct) + { + _mintStarted.TrySetResult(); + await _allowMint.Task.WaitAsync(ct); + return credential; + } + + public Task RevokeAsync(string accessToken, CancellationToken ct) + { + RevokedTokens.Add(accessToken); + return Task.CompletedTask; + } + + public void AllowMint() => _allowMint.TrySetResult(); + } + private sealed class MutableTimeProvider(DateTimeOffset utcNow) : TimeProvider { private DateTimeOffset _utcNow = utcNow; From e899817b9842796672a1c21a26521d3500ac9198 Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 22:17:25 -0700 Subject: [PATCH 10/12] fix: require approval for repository workflow runs Refs #947 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- docs/deep-dive/sandboxed-execution.md | 6 +-- packages/Agentweaver.Domain/SandboxPolicy.cs | 1 + .../AssemblyBuildTestShellGuardTests.cs | 49 +++++++++++++++++++ .../Sandbox/SandboxPolicyPreserveTests.cs | 2 + 4 files changed, 55 insertions(+), 3 deletions(-) diff --git a/docs/deep-dive/sandboxed-execution.md b/docs/deep-dive/sandboxed-execution.md index 2aaa2a49b..d9fbb6fd8 100644 --- a/docs/deep-dive/sandboxed-execution.md +++ b/docs/deep-dive/sandboxed-execution.md @@ -106,9 +106,9 @@ browser/editor forms are rejected even after operator approval, so they never re start an external program. Likewise, `gh gist edit` and every other editor-facing gist form remain outside the allowlist. -The approval policy gates `git push`, remote changes, `gh pr` changes, `gh repo` changes, `gh api`, -`gh secret set`, and `gh auth` commands (including `gh auth token`). The API does not inspect or -proxy these commands. +The approval policy gates `git push`, remote changes, `gh pr` changes, `gh repo` changes, +`gh workflow run`, `gh api`, `gh secret set`, and `gh auth` commands (including `gh auth token`). +The API does not inspect or proxy these commands. The system keeps this credential out of pod specs, files, logs, events, annotations, shared environments, and credential-helper files. Normal release and orphan cleanup log failed revocations. The registry retains failed revocations independently of their SandboxClaim, and each reaper sweep retries diff --git a/packages/Agentweaver.Domain/SandboxPolicy.cs b/packages/Agentweaver.Domain/SandboxPolicy.cs index 64354761b..8275f0eed 100644 --- a/packages/Agentweaver.Domain/SandboxPolicy.cs +++ b/packages/Agentweaver.Domain/SandboxPolicy.cs @@ -69,6 +69,7 @@ public sealed record SandboxPolicy // GitHub repository changes and credential commands "gh pr create", "gh pr merge", "gh pr close", "gh repo delete", "gh repo archive", + "gh workflow run", "gh api", "gh secret set", "gh auth login", "gh auth logout", "gh auth token", // PowerShell destructive "Remove-Item -Recurse", "Remove-Item -Force", "ri -r", "ri -Recurse", diff --git a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs index f37845ac1..0b1a8a6c2 100644 --- a/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/AssemblyBuildTestShellGuardTests.cs @@ -335,6 +335,55 @@ public async Task Controlled_run_command_requires_operator_approval_for_sensitiv executor.ExecuteCalls.Should().Be(0); } + [Fact] + public async Task Controlled_run_command_requires_approval_before_direct_credentialed_workflow_run() + { + const string command = "gh workflow run ci.yml --repo sabbour/agentweaver"; + const string sentinel = "repository-access-token"; + var approved = false; + var approvalChecks = 0; + SandboxCommand? observed = null; + var executor = new CapturingExecutor(candidate => observed = candidate); + using var tracker = new ShellExecutionTracker(); + var context = BuildContext( + executor, + tracker, + repositoryAccessToken: sentinel, + destructivePatterns: [.. SandboxPolicy.Default(_root).DestructiveCommandPatterns], + rejectDestructiveCommands: false, + isCommandApproved: _ => + { + approvalChecks++; + return approved; + }); + var tool = CopilotAIAgent.BuildSessionConfigTools( + context, + includeControlledRunCommand: true).Single(t => t.Name == "run_command"); + + var awaitingApproval = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = command })); + + awaitingApproval?.ToString().Should().Contain("requires operator approval"); + approvalChecks.Should().Be(1); + observed.Should().BeNull("the workflow must not start before its command is approved"); + + approved = true; + var executed = await tool.InvokeAsync(new AIFunctionArguments( + new Dictionary { ["command"] = command })); + + executed?.ToString().Should().Contain("exit_code: 0"); + approvalChecks.Should().Be(2); + observed.Should().NotBeNull(); + observed!.CommandLine.Should().NotContain(sentinel); + observed.Environment.Should().NotContain(pair => pair.Value == sentinel); + observed.DirectExecution.Should().NotBeNull(); + observed.DirectExecution!.Executable.Should().Be("gh"); + observed.DirectExecution.Arguments.Should().Equal( + "workflow", "run", "ci.yml", "--repo", "sabbour/agentweaver"); + observed.DirectExecution.Environment.Should().Contain( + new KeyValuePair("GH_TOKEN", sentinel)); + } + [Theory] [InlineData("gh 'secret' set DEPLOY_KEY --body value")] [InlineData("gh \"secret\" set DEPLOY_KEY --body value")] diff --git a/tests/Agentweaver.Tests/Sandbox/SandboxPolicyPreserveTests.cs b/tests/Agentweaver.Tests/Sandbox/SandboxPolicyPreserveTests.cs index fc6b621d5..352a4113c 100644 --- a/tests/Agentweaver.Tests/Sandbox/SandboxPolicyPreserveTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/SandboxPolicyPreserveTests.cs @@ -235,6 +235,7 @@ await WriteSettingsAsync( .GetPolicyAsync(_repoPath); policy.DestructiveCommandPatterns.Should().Equal("user-defined-command"); + policy.DestructiveCommandPatterns.Should().NotContain("gh workflow run"); } // ── Helpers ───────────────────────────────────────────────────────────────────────────────── @@ -324,6 +325,7 @@ private static async Task AssertCanonicalCommandsRequireApprovalAsync(SandboxPol "gh pr close 1", "gh repo delete example/repo", "gh repo archive example/repo", + "gh workflow run ci.yml --repo sabbour/agentweaver", }) { var result = await tool.InvokeAsync(new AIFunctionArguments( From c2740dedb87c719082569feec97d6c175e85db3d Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 22:32:44 -0700 Subject: [PATCH 11/12] fix: reconcile repository credentials across replicas Reconcile each API replica's local credential registry with shared run and SandboxClaim state so releases and orphan cleanup on another replica trigger local revocation retries. Refs #947 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- apps/Agentweaver.Api/Program.cs | 2 + ...positoryCredentialReconciliationService.cs | 205 ++++++++++++++++++ .../RunRepositoryCredentialRegistry.cs | 64 ++++++ ...oryCredentialReconciliationServiceTests.cs | 194 +++++++++++++++++ 4 files changed, 465 insertions(+) create mode 100644 apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialReconciliationService.cs create mode 100644 tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialReconciliationServiceTests.cs diff --git a/apps/Agentweaver.Api/Program.cs b/apps/Agentweaver.Api/Program.cs index 6fec6512d..e0b756c8c 100644 --- a/apps/Agentweaver.Api/Program.cs +++ b/apps/Agentweaver.Api/Program.cs @@ -487,6 +487,8 @@ // instead of the installation token (which fails the first model turn). builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddHostedService(); builder.Services.AddSingleton(sp => new RunAgentHostContextResolver( sp.GetRequiredService(), diff --git a/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialReconciliationService.cs b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialReconciliationService.cs new file mode 100644 index 000000000..071ee9b9f --- /dev/null +++ b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialReconciliationService.cs @@ -0,0 +1,205 @@ +using Agentweaver.Api.Infrastructure; +using Agentweaver.Domain; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; + +namespace Agentweaver.Api.Sandbox; + +/// +/// Resolves whether this replica's locally held repository credentials still have a live owner. +/// +internal interface IRunRepositoryCredentialLiveness +{ + Task> GetTerminalOrGoneRunIdsAsync( + IReadOnlyList runIds, + CancellationToken ct = default); +} + +/// +/// Reads the shared run store and cluster claim inventory. No repository credentials cross this +/// boundary: only locally held run identifiers are reconciled against durable/cluster state. +/// +internal sealed class RunRepositoryCredentialLiveness : IRunRepositoryCredentialLiveness +{ + private readonly IRunStore _runStore; + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public RunRepositoryCredentialLiveness( + IRunStore runStore, + IServiceProvider services, + ILogger logger) + { + _runStore = runStore; + _services = services; + _logger = logger; + } + + public async Task> GetTerminalOrGoneRunIdsAsync( + IReadOnlyList runIds, + CancellationToken ct = default) + { + var terminalOrGone = new HashSet(StringComparer.Ordinal); + var claimsToVerify = new Dictionary(StringComparer.Ordinal); + + foreach (var runId in runIds) + { + ct.ThrowIfCancellationRequested(); + + if (!RunId.TryParse(runId, out var parsedRunId)) + { + // Run IDs are UUIDs by domain invariant, so an unparseable owner cannot be live. + terminalOrGone.Add(runId); + continue; + } + + Run? run; + try + { + run = await _runStore.GetAsync(parsedRunId, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Repository credential reconciliation could not read run {RunId}; retaining local credential until the next sweep", + runId); + continue; + } + + if (run is null || IsTerminal(run.Status)) + { + terminalOrGone.Add(runId); + continue; + } + + if (!string.IsNullOrWhiteSpace(run.SandboxClaimName)) + claimsToVerify.Add(runId, run.SandboxClaimName); + } + + if (claimsToVerify.Count == 0) + return terminalOrGone; + + var reaper = _services.GetService(); + if (reaper is null) + return terminalOrGone; + + IReadOnlyList inventory; + try + { + inventory = await reaper.GetClaimInventoryAsync(ct).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Repository credential reconciliation could not read SandboxClaim inventory; retaining local credentials until the next sweep"); + return terminalOrGone; + } + + foreach (var (runId, claimName) in claimsToVerify) + { + var claimIsLive = inventory.Any(claim => + string.Equals(claim.ClaimName, claimName, StringComparison.Ordinal) && + string.Equals(claim.AnnotatedRunId, runId, StringComparison.Ordinal)); + if (!claimIsLive) + terminalOrGone.Add(runId); + } + + return terminalOrGone; + } + + private static bool IsTerminal(RunStatus status) => status is + RunStatus.Completed or RunStatus.Failed or RunStatus.Merged or RunStatus.Declined or + RunStatus.MergeFailed or RunStatus.AssembleReady; +} + +/// +/// Each API replica periodically reconciles its own in-memory repository credentials against +/// shared run/claim state. This closes the release path where a different replica deletes the +/// SandboxClaim and therefore cannot see or revoke this replica's token. +/// +internal sealed class RunRepositoryCredentialReconciliationService : BackgroundService +{ + private static readonly TimeSpan SweepInterval = TimeSpan.FromSeconds(30); + + private readonly RunRepositoryCredentialRegistry _registry; + private readonly IRunRepositoryCredentialLiveness _liveness; + private readonly ILogger _logger; + + public RunRepositoryCredentialReconciliationService( + RunRepositoryCredentialRegistry registry, + IRunRepositoryCredentialLiveness liveness, + ILogger logger) + { + _registry = registry; + _liveness = liveness; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + await ReconcileOnceAsync(stoppingToken).ConfigureAwait(false); + + try + { + await Task.Delay(SweepInterval, stoppingToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (stoppingToken.IsCancellationRequested) + { + break; + } + } + } + + /// + /// Performs one local-registry reconciliation. Extracted for focused replica-lifecycle tests. + /// + internal async Task ReconcileOnceAsync(CancellationToken ct = default) + { + IReadOnlySet terminalOrGone = new HashSet(StringComparer.Ordinal); + var activeRunIds = _registry.GetActiveCredentialRunIds(); + + if (activeRunIds.Count > 0) + { + try + { + terminalOrGone = await _liveness + .GetTerminalOrGoneRunIdsAsync(activeRunIds, ct) + .ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning( + ex, + "Repository credential reconciliation could not determine credential liveness; retrying next sweep"); + } + } + + var failures = await _registry + .ReconcileTerminalOrGoneAsync(terminalOrGone, ct) + .ConfigureAwait(false); + foreach (var failure in failures) + { + _logger.LogWarning( + failure.Exception, + "Repository credential reconciliation failed to revoke credential for run {RunId}; it will retry with backoff until expiry", + failure.RunId); + } + } +} diff --git a/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs index 797198509..aa20b9b16 100644 --- a/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs +++ b/apps/Agentweaver.Api/Sandbox/RunRepositoryCredentialRegistry.cs @@ -33,6 +33,12 @@ internal int ActiveRunLockCount } } + /// + /// Returns this replica's locally minted credential owners. The access tokens stay + /// private to this registry; callers receive only run identifiers for liveness reconciliation. + /// + internal IReadOnlyList GetActiveCredentialRunIds() => _entries.Keys.ToArray(); + public RunRepositoryCredentialRegistry(IServiceScopeFactory scopeFactory) : this(new RunRepositoryCredentialMinter(scopeFactory)) { @@ -173,6 +179,64 @@ internal async Task> RetryFa return failures; } + /// + /// Revokes credentials this replica minted for runs whose durable run or SandboxClaim state has + /// become terminal or disappeared. Failed provider revocations remain in the in-memory retry + /// set until expiry, regardless of whether another replica deleted the claim that prompted this + /// reconciliation. + /// + internal async Task> ReconcileTerminalOrGoneAsync( + IReadOnlySet terminalOrGoneRunIds, + CancellationToken ct = default) + { + var failures = new List(); + var activeRunIds = _entries.Keys.ToArray(); + foreach (var runId in activeRunIds) + { + ct.ThrowIfCancellationRequested(); + + if (!terminalOrGoneRunIds.Contains(runId)) + { + await RemoveExpiredEntryAsync(runId, ct).ConfigureAwait(false); + continue; + } + + try + { + await RevokeAsync(runId, ct).ConfigureAwait(false); + } + catch (OperationCanceledException) when (ct.IsCancellationRequested) + { + throw; + } + catch (Exception ex) + { + failures.Add(new FailedRepositoryCredentialRevocation(runId, ex)); + } + } + + failures.AddRange(await RetryFailedRevocationsAsync(ct).ConfigureAwait(false)); + return failures; + } + + private async Task RemoveExpiredEntryAsync(string runId, CancellationToken ct) + { + using var mintLockLease = AcquireMintLock(runId); + await mintLockLease.Semaphore.WaitAsync(ct).ConfigureAwait(false); + try + { + if (_entries.TryGetValue(runId, out var entry) && + entry.ExpiresAt <= _timeProvider.GetUtcNow()) + { + _entries.TryRemove(runId, out _); + } + } + finally + { + mintLockLease.Semaphore.Release(); + } + } + private RunCredentialLockLease AcquireMintLock(string runId) { lock (_mintLockGate) diff --git a/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialReconciliationServiceTests.cs b/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialReconciliationServiceTests.cs new file mode 100644 index 000000000..49e83d099 --- /dev/null +++ b/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialReconciliationServiceTests.cs @@ -0,0 +1,194 @@ +using System.Net.Http; +using Agentweaver.Api.Infrastructure; +using Agentweaver.Api.Sandbox; +using Agentweaver.Domain; +using Agentweaver.Tests.Helpers; +using FluentAssertions; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Agentweaver.Tests.Sandbox; + +public sealed class RunRepositoryCredentialReconciliationServiceTests +{ + [Fact] + public async Task ReconcileOnce_RevokesReplicaACredential_WhenReplicaBReleasesClaim() + { + await using var database = await TestSqliteDb.CreateAsync(); + var runStore = new SqliteRunStore(database.Db); + var run = await InsertRunAsync(runStore, RunStatus.AwaitingReview); + var claims = new ReplicaBClaimStore(); + claims.Add(run.SandboxClaimName!, run.Id.ToString()); + using var services = CreateServices(claims); + + var minter = new StubCredentialMinter( + new RepositoryCredential("replica-a-release-token", DateTimeOffset.UtcNow.AddMinutes(5))); + var registry = new RunRepositoryCredentialRegistry(minter); + (await registry.MintAsync(run.Id.ToString())).Should().Be("replica-a-release-token"); + var replicaA = CreateReconciler(registry, runStore, services); + + // Replica B performs the normal pod release after the run reaches its review state. + claims.DeleteClaimFromReplicaB(run.SandboxClaimName!); + + await replicaA.ReconcileOnceAsync(); + + minter.RevokedTokens.Should().ContainSingle().Which.Should().Be("replica-a-release-token", + "replica A must observe replica B's authoritative claim deletion on its next sweep"); + } + + [Fact] + public async Task ReconcileOnce_RetriesRevocationAfterReplicaBDeletesRunAndClaim() + { + var now = new DateTimeOffset(2026, 8, 27, 20, 0, 0, TimeSpan.Zero); + await using var database = await TestSqliteDb.CreateAsync(); + var runStore = new SqliteRunStore(database.Db); + var run = await InsertRunAsync(runStore, RunStatus.InProgress); + var claims = new ReplicaBClaimStore(); + claims.Add(run.SandboxClaimName!, run.Id.ToString()); + using var services = CreateServices(claims); + + var minter = new StubCredentialMinter( + new RepositoryCredential("replica-a-retry-token", now.AddMinutes(5))); + minter.RevokeFailures.Enqueue(new HttpRequestException("temporary revoke failure")); + var clock = new MutableTimeProvider(now); + var registry = new RunRepositoryCredentialRegistry(minter, clock); + (await registry.MintAsync(run.Id.ToString())).Should().Be("replica-a-retry-token"); + var replicaA = CreateReconciler(registry, runStore, services); + + // The orphan-cleanup request landed on replica B, which removed both shared records. + await runStore.DeleteAsync(run.Id); + claims.DeleteClaimFromReplicaB(run.SandboxClaimName!); + + await replicaA.ReconcileOnceAsync(); + minter.RevokedTokens.Should().ContainSingle( + "the first failed revoke is retained locally even after replica B removed the claim"); + + clock.Advance(RunRepositoryCredentialRegistry.InitialRevocationRetryDelay); + await replicaA.ReconcileOnceAsync(); + + minter.RevokedTokens.Should().HaveCount(2, + "the retained credential is retried through its expiry even though its run and claim are gone"); + minter.RevokedTokens.Should().OnlyContain(token => token == "replica-a-retry-token"); + } + + [Fact] + public async Task ReconcileOnce_DoesNotRevokeCredentialForActiveRunWithLiveClaim() + { + await using var database = await TestSqliteDb.CreateAsync(); + var runStore = new SqliteRunStore(database.Db); + var run = await InsertRunAsync(runStore, RunStatus.InProgress); + var claims = new ReplicaBClaimStore(); + claims.Add(run.SandboxClaimName!, run.Id.ToString()); + using var services = CreateServices(claims); + + var minter = new StubCredentialMinter( + new RepositoryCredential("active-run-token", DateTimeOffset.UtcNow.AddMinutes(5))); + var registry = new RunRepositoryCredentialRegistry(minter); + (await registry.MintAsync(run.Id.ToString())).Should().Be("active-run-token"); + + await CreateReconciler(registry, runStore, services).ReconcileOnceAsync(); + + minter.RevokedTokens.Should().BeEmpty( + "a non-terminal run whose authoritative AgentHost claim is still present remains live"); + } + + [Fact] + public void ApiRegistersRepositoryCredentialReconciliationAsHostedService() + { + using var factory = new AgentweaverWebApplicationFactory(); + + factory.Services.GetServices() + .Should().Contain(service => service is RunRepositoryCredentialReconciliationService); + } + + private static ServiceProvider CreateServices(IAgentHostReaper reaper) => + new ServiceCollection() + .AddSingleton(reaper) + .BuildServiceProvider(); + + private static RunRepositoryCredentialReconciliationService CreateReconciler( + RunRepositoryCredentialRegistry registry, + IRunStore runStore, + IServiceProvider services) => + new( + registry, + new RunRepositoryCredentialLiveness( + runStore, + services, + NullLogger.Instance), + NullLogger.Instance); + + private static async Task InsertRunAsync(SqliteRunStore runStore, RunStatus status) + { + var id = RunId.New(); + var claimName = SandboxClaimConventions.DeriveAgentHostClaimName(id.ToString()); + var run = new Run + { + Id = id, + RepositoryPath = "credential-reconciliation-repo", + OriginatingBranch = "main", + ModelSource = ModelSource.GitHubCopilot, + Task = "credential reconciliation", + SubmittingUser = "replica-test-user", + Status = status, + StartedAt = DateTimeOffset.UtcNow, + SandboxBackend = "kubernetes-sandbox-claim", + SandboxClaimName = claimName, + SandboxNamespace = "agentweaver", + }; + await runStore.InsertAsync(run); + return run; + } + + private sealed class ReplicaBClaimStore : IAgentHostReaper + { + private readonly Dictionary _claims = new(StringComparer.Ordinal); + + public void Add(string claimName, string runId) => + _claims[claimName] = new AgentHostClaimInfo( + claimName, + RunId: runId, + PodName: "agenthost-pod", + Ready: true, + CreatedAt: DateTimeOffset.UtcNow, + Orphaned: false, + AnnotatedRunId: runId); + + public void DeleteClaimFromReplicaB(string claimName) => _claims.Remove(claimName); + + public Task SweepOrphanedPodsAsync(CancellationToken ct = default) => + Task.FromResult(0); + + public Task> GetClaimInventoryAsync( + CancellationToken ct = default) => + Task.FromResult>(_claims.Values.ToArray()); + } + + private sealed class StubCredentialMinter(RepositoryCredential credential) + : IRunRepositoryCredentialMinter + { + public Queue RevokeFailures { get; } = new(); + public List RevokedTokens { get; } = []; + + public Task MintAsync(string runId, CancellationToken ct) => + Task.FromResult(credential); + + public Task RevokeAsync(string accessToken, CancellationToken ct) + { + RevokedTokens.Add(accessToken); + if (RevokeFailures.TryDequeue(out var failure)) + throw failure; + return Task.CompletedTask; + } + } + + private sealed class MutableTimeProvider(DateTimeOffset utcNow) : TimeProvider + { + private DateTimeOffset _utcNow = utcNow; + + public override DateTimeOffset GetUtcNow() => _utcNow; + + public void Advance(TimeSpan duration) => _utcNow += duration; + } +} From fa1df3f64f2ff6d715363112d57a1f8730fc04af Mon Sep 17 00:00:00 2001 From: sabbour Date: Thu, 27 Aug 2026 23:16:35 -0700 Subject: [PATCH 12/12] fix: persist real AgentHost claim name for cross-replica revocation LaunchAgentHostPodAsync derived its own authoritative AgentHost SandboxClaim name but never wrote it back to the run store, so kata-exec-sidecar AgentHost runs always had Run.SandboxClaimName unset. RunRepositoryCredentialLiveness only checks claim liveness when that field is populated, so a replica that minted a repository credential for an AgentHost run could never learn another replica had deleted its claim, and the credential was never revoked. Persist the claim name (and the real kata-exec-sidecar backend, now a shared ExecutorBackendName constant instead of an inferred label) from inside LaunchAgentHostPodAsync itself, immediately after the claim is created or reclaimed and before any repository credential can be minted. The write uses SetSandboxInfoAsync's existing non-destructive partial-update semantics, and the deterministic claim-name derivation means restart/reclaim always restores the same name. Replace the prior unit test's manual SandboxClaimName injection with an integration-style test that drives the real LaunchAgentHostPodAsync claim lifecycle against a fake cluster, asserts production code (not test setup) persisted the claim, then simulates a second replica deleting that claim and proves the first replica revokes its local token. Refs #947 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 3f94bc67-d29d-4d47-a8f7-5d99540c4b42 --- .../Sandbox/KubernetesSandboxExecutor.cs | 64 ++++++++++- .../Sandbox/SandboxExecutorRouter.cs | 8 +- .../PodExec/PodExecSandboxClient.cs | 11 +- ...oryCredentialReconciliationServiceTests.cs | 104 ++++++++++++++++++ 4 files changed, 183 insertions(+), 4 deletions(-) diff --git a/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs b/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs index 4165f795c..2b45fde32 100644 --- a/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs +++ b/apps/Agentweaver.Api/Sandbox/KubernetesSandboxExecutor.cs @@ -11,6 +11,7 @@ using k8s; using k8s.Autorest; using Agentweaver.SandboxExec; +using Agentweaver.SandboxExec.PodExec; using Microsoft.Extensions.Logging; namespace Agentweaver.Api.Sandbox; @@ -221,6 +222,13 @@ internal sealed class KubernetesSandboxExecutor : ISandboxExecutor, IAgentHostPo // Previewable/PreviewActive state and applies all retention or cleanup effects before deciding // whether to delete the claim. private readonly Agentweaver.Api.Sandbox.Preview.ISandboxPreviewService? _previewService; + // Persists the AUTHORITATIVE AgentHost claim name (LaunchAgentHostPodAsync's own + // SandboxClaimConventions.DeriveAgentHostClaimName derivation) into Run.SandboxClaimName as soon + // as the claim is created/reclaimed — before any repository credential can be minted — so + // cross-replica liveness reconciliation (RunRepositoryCredentialLiveness) can always resolve this + // run's real cluster claim. Null in unit tests → the persistence is skipped (same null-skip + // convention as the other optional collaborators above). + private readonly IRunStore? _runStore; public bool IsRealIsolation => true; public string BackendName => "kubernetes-sandbox-claim"; @@ -246,7 +254,8 @@ internal KubernetesSandboxExecutor( IGitHubAccessTokenProvider? accessTokenProvider = null, Agentweaver.Api.Sandbox.Preview.ISandboxPreviewService? previewService = null, IGitHubTokenScopeProvider? tokenScopeProvider = null, - Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null) + Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null, + IRunStore? runStore = null) { _client = client; _options = options; @@ -265,6 +274,7 @@ internal KubernetesSandboxExecutor( _accessTokenProvider = accessTokenProvider; _previewService = previewService; _authorshipCapabilityStore = authorshipCapabilityStore; + _runStore = runStore; } public async Task ExecuteAsync( @@ -499,6 +509,14 @@ public async Task LaunchAgentHostPodAsync( } } + // The claim now definitely exists on the cluster — either just created or reclaimed + // (reused) above. Persist ITS OWN authoritative claim name into Run.SandboxClaimName + // right now, before anything below can mint a repository credential, so cross-replica + // liveness reconciliation always has a real claim to check (issue: kata-exec-sidecar + // AgentHost runs previously left SandboxClaimName unset, so a replica that minted a + // repository credential could never learn that another replica deleted the claim). + await PersistAgentHostClaimNameAsync(runId, claimName, ct).ConfigureAwait(false); + var podName = await WaitForBoundWithProvisioningHeartbeatAsync(runId, claimName, ct).ConfigureAwait(false); _logger.LogInformation( "KubernetesSandboxExecutor: AgentHost claim {Claim} bound to pod {Pod}", claimName, podName); @@ -612,6 +630,50 @@ await _authorshipCapabilityStore.RemoveAsync(runId, CancellationToken.None) } } + /// + /// Persists 's + /// own authoritative (the SAME value it just created or reclaimed on + /// the cluster) into Run.SandboxClaimName/SandboxBackend/SandboxNamespace. + /// + /// + /// This is the ONLY place that should ever write the AgentHost claim identity for a run: + /// later reads it back from the shared run store to + /// confirm a non-terminal run's claim is still present in the cluster's SandboxClaim inventory. + /// Without this write (the PR #968 gap), kata-exec-sidecar AgentHost runs never had a + /// persisted claim name, so a replica that minted a repository credential could never learn that + /// another replica later deleted the claim, and the credential was never revoked. + /// + /// + /// + /// Deliberately fails the launch (rather than degrading silently, unlike the other optional + /// collaborators in this class) when the run id cannot be parsed: minting a repository credential + /// for a run whose claim identity can never be resolved back would make that credential + /// unrevocable via cross-replica reconciliation. A null (unit tests) still + /// degrades to a no-op, matching this class's established null-skip convention. + /// + /// + private async Task PersistAgentHostClaimNameAsync(string runId, string claimName, CancellationToken ct) + { + if (_runStore is null) + return; + + if (!RunId.TryParse(runId, out var parsedRunId)) + { + throw new InvalidOperationException( + $"Cannot persist AgentHost claim '{claimName}' for run '{runId}': the run id does not " + + "parse as a RunId, so cross-replica credential-liveness reconciliation could never " + + "resolve this run's claim back from the shared run store."); + } + + await _runStore.SetSandboxInfoAsync( + parsedRunId, + PodExecSandboxClient.ExecutorBackendName, + claimName, + podName: null, + @namespace: _options.Namespace, + ct).ConfigureAwait(false); + } + /// public async Task ReleaseAgentHostPodAsync(string runId, CancellationToken ct = default) { diff --git a/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs b/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs index ccd1e294f..ba26c588d 100644 --- a/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs +++ b/apps/Agentweaver.Api/Sandbox/SandboxExecutorRouter.cs @@ -33,6 +33,7 @@ public sealed class SandboxExecutorRouter : ISandboxExecutorRouter private readonly IGitHubAccessTokenProvider? _accessTokenProvider; private readonly Preview.ISandboxPreviewService? _previewService; private readonly RunRepositoryCredentialRegistry? _repositoryCredentials; + private readonly IRunStore? _runStore; public SandboxExecutorRouter(IConfiguration config, ILoggerFactory loggerFactory, IPodNameRegistry? podRegistry = null, IHttpClientFactory? httpClientFactory = null, @@ -46,7 +47,8 @@ public SandboxExecutorRouter(IConfiguration config, ILoggerFactory loggerFactory IGitHubAccessTokenProvider? accessTokenProvider = null, Preview.ISandboxPreviewService? previewService = null, RunRepositoryCredentialRegistry? repositoryCredentials = null, - Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null) + Security.IRunAuthorshipCapabilityStore? authorshipCapabilityStore = null, + IRunStore? runStore = null) { _config = config; _loggerFactory = loggerFactory; @@ -63,6 +65,7 @@ public SandboxExecutorRouter(IConfiguration config, ILoggerFactory loggerFactory _previewService = previewService; _repositoryCredentials = repositoryCredentials; _authorshipCapabilityStore = authorshipCapabilityStore; + _runStore = runStore; } public ISandboxExecutor Resolve() @@ -150,7 +153,8 @@ public ISandboxExecutor Resolve() _submittingUserResolver, _httpClientFactory, _tokenStore, _secretStore, _runEventStream, _runOptions, _repositoryCredentials, _accessTokenProvider, _previewService, tokenScopeProvider: _tokenScopeProvider, - authorshipCapabilityStore: _authorshipCapabilityStore); + authorshipCapabilityStore: _authorshipCapabilityStore, + runStore: _runStore); } catch (Exception ex) { diff --git a/packages/Agentweaver.SandboxExec/PodExec/PodExecSandboxClient.cs b/packages/Agentweaver.SandboxExec/PodExec/PodExecSandboxClient.cs index b21cf3296..219f409e2 100644 --- a/packages/Agentweaver.SandboxExec/PodExec/PodExecSandboxClient.cs +++ b/packages/Agentweaver.SandboxExec/PodExec/PodExecSandboxClient.cs @@ -34,8 +34,17 @@ public PodExecSandboxClient( _relayAssembly = relayAssembly ?? System.Reflection.Assembly.GetEntryAssembly()?.Location ?? string.Empty; } + /// + /// Backend identifier this executor reports on every sandbox.selected event for AgentHost + /// pod-per-run command execution. Exposed as a constant (not just the instance property below) + /// so KubernetesSandboxExecutor.LaunchAgentHostPodAsync can persist the SAME literal into + /// Run.SandboxBackend when it creates/reclaims the AgentHost claim, instead of a caller + /// re-deriving it from a downstream event/backend label (PR #968 cross-replica repair). + /// + public const string ExecutorBackendName = "kata-exec-sidecar"; + public bool IsRealIsolation => true; - public string BackendName => "kata-exec-sidecar"; + public string BackendName => ExecutorBackendName; public string SelectionReason => "Kata VM plus a dedicated executor sidecar container (own PID/mount namespace) running a fail-closed bubblewrap mount namespace per run."; public bool HasNetworkWarning => false; diff --git a/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialReconciliationServiceTests.cs b/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialReconciliationServiceTests.cs index 49e83d099..25a2a5408 100644 --- a/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialReconciliationServiceTests.cs +++ b/tests/Agentweaver.Tests/Sandbox/RunRepositoryCredentialReconciliationServiceTests.cs @@ -4,6 +4,7 @@ using Agentweaver.Domain; using Agentweaver.Tests.Helpers; using FluentAssertions; +using k8s; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging.Abstractions; @@ -93,6 +94,109 @@ public async Task ReconcileOnce_DoesNotRevokeCredentialForActiveRunWithLiveClaim "a non-terminal run whose authoritative AgentHost claim is still present remains live"); } + /// + /// Regression for PR #968's rejected cross-replica repair: the tests above insert a run with + /// SandboxClaimName set BY THE TEST, which hid the real production gap — the + /// kata-exec-sidecar AgentHost lifecycle () + /// never persisted the claim name it created, so this reconciliation never had anything to check. + /// This test drives that REAL lifecycle end-to-end — no manual SandboxClaimName injection — + /// against a fake cluster, then simulates a second replica deleting the same claim, and proves the + /// first replica revokes its locally held repository credential. + /// + [Fact] + public async Task ReconcileOnce_RevokesReplicaACredential_WhenReplicaBReleasesTheRealClaimLaunchAgentHostPodAsyncPersisted() + { + var id = RunId.New(); + var runId = id.ToString(); + var claimName = SandboxClaimConventions.DeriveAgentHostClaimName(runId); + + await using var database = await TestSqliteDb.CreateAsync(); + var runStore = new SqliteRunStore(database.Db); + + // Deliberately NO SandboxBackend/SandboxClaimName/SandboxNamespace here — proving those are + // written by LaunchAgentHostPodAsync itself, not by test setup. + var run = new Run + { + Id = id, + RepositoryPath = "real-claim-lifecycle-repo", + OriginatingBranch = "main", + ModelSource = ModelSource.GitHubCopilot, + Task = "real AgentHost claim lifecycle", + SubmittingUser = "replica-a-user", + Status = RunStatus.InProgress, + StartedAt = DateTimeOffset.UtcNow, + }; + await runStore.InsertAsync(run); + + // Fakes the cluster's view of the AgentHost claim/pod once the agent-sandbox controller has + // bound it — the same shape KubernetesSandboxExecutor's real WaitForBound polling expects. + var kube = new FakeKubeHandler(); + kube.OnGet( + $"/apis/{SandboxClaimConventions.ApiGroup}/{SandboxClaimConventions.ApiVersion}/namespaces/agentweaver/sandboxclaims/{claimName}", + """{"status":{"conditions":[{"type":"Ready","status":"True"}],"sandbox":{"name":"agent-real-claim-pod"}}}"""); + kube.OnAny( + @"^/api/v1/namespaces/agentweaver/pods/agent-real-claim-pod$", + """{"kind":"Pod","metadata":{"name":"agent-real-claim-pod"},"status":{"podIP":"10.0.5.9"}}"""); + + var k8sClient = new Kubernetes( + new KubernetesClientConfiguration { Host = "http://localhost:8080" }, kube); + var options = new KubernetesSandboxOptions + { + Namespace = "agentweaver", + WarmPoolRef = "agentweaver-sandbox", + AgentHostWarmPoolRef = "agentweaver-agent-host", + TimeoutSeconds = 600, + RequireMtls = false, + AgentHostPort = 8088, + AgentHostA2APath = "/a2a/agent", + WorkspaceMountPath = "/workspace", + }; + + // RunStoreSubmittingUserResolver is the REAL production resolver (reads Run.SubmittingUser + // back from the same store), not a hand-rolled stub — keeping this launch as close to + // production as the fake cluster allows. runStore is wired in so LaunchAgentHostPodAsync can + // exercise its own persistence write. + var executor = new KubernetesSandboxExecutor( + k8sClient, + options, + NullLogger.Instance, + submittingUserResolver: new RunStoreSubmittingUserResolver(runStore), + runStore: runStore); + + var endpoint = await executor.LaunchAgentHostPodAsync(runId); + endpoint.Should().Contain("10.0.5.9"); + + // Proves the fix: production code — not this test — wrote the real claim identity. + var persistedRun = await runStore.GetAsync(id); + persistedRun.Should().NotBeNull(); + persistedRun!.SandboxClaimName.Should().Be(claimName); + persistedRun.SandboxBackend.Should().Be("kata-exec-sidecar"); + + // Replica A mints and locally holds a repository credential for this run (decoupled from the + // executor's own minting path, mirroring how the other tests in this file isolate the + // registry/reconciliation behavior under test). + var minter = new StubCredentialMinter( + new RepositoryCredential("replica-a-real-claim-token", DateTimeOffset.UtcNow.AddMinutes(5))); + var registry = new RunRepositoryCredentialRegistry(minter); + (await registry.MintAsync(runId)).Should().Be("replica-a-real-claim-token"); + + // Replica B's cluster view initially matches the claim LaunchAgentHostPodAsync created... + var claims = new ReplicaBClaimStore(); + claims.Add(claimName, runId); + using var services = CreateServices(claims); + var replicaA = CreateReconciler(registry, runStore, services); + + // ...until replica B performs the normal pod release and deletes it. + claims.DeleteClaimFromReplicaB(claimName); + + await replicaA.ReconcileOnceAsync(); + + minter.RevokedTokens.Should().ContainSingle().Which.Should().Be("replica-a-real-claim-token", + "replica A must revoke its local token once replica B deletes the REAL AgentHost claim " + + "that LaunchAgentHostPodAsync itself created and persisted, with no manual " + + "SandboxClaimName injection standing in for production behavior"); + } + [Fact] public void ApiRegistersRepositoryCredentialReconciliationAsHostedService() {