diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 57b3bfa1a..67bee9ddb 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -122,6 +122,37 @@ jobs: *) echo "cli/package.json version must be semver, got $head_version" >&2; exit 1 ;; esac + coauthor-trailers: + name: Co-author trailers + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + with: + fetch-depth: 0 + persist-credentials: false + - name: Reject GitHub noreply addresses in Co-Authored-By trailers + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + set -euo pipefail + base=$(git merge-base "$BASE_SHA" "$HEAD_SHA") + offenders="" + for sha in $(git rev-list "$base..$HEAD_SHA"); do + if git log -1 --format='%(trailers:key=Co-authored-by,valueonly)' "$sha" \ + | grep -qi '@users\.noreply\.github\.com'; then + offenders="$offenders $sha" + fi + done + [ -n "$offenders" ] || exit 0 + echo "$offenders" | xargs git show -s --format='%h %s' >&2 + echo "Co-Authored-By trailers must not use @users.noreply.github.com addresses:" >&2 + echo "GitHub credits them to whichever account owns that username, which may be a stranger." >&2 + echo "This repo bans all such addresses, including your own privacy address;" >&2 + echo "use the contributor's real email, or drop the trailer." >&2 + exit 1 + lint: name: Lint runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index 114fba427..d3661e12d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -59,11 +59,14 @@ Two habits that keep task-focused changes from scarring the rest of the repo: Slack Mac app, and don't ask permission first — do it on your own; don't wait to be asked. Skip it for trivial refactors, docs, config, or pure-logic changes already covered by tests. -- **Screenshot every front-end change in the PR.** Anything an operator or user sees - rendered — admin/web/portal UI, Slack surfaces, emails — ships with a screenshot of the - after state (before/after when it's a change to something that already existed) in the PR - description, so a reviewer sees the result without booting it. Can't reach the surface - live? Render it against realistic data and say so. +- **Demo every front-end change in the PR.** Anything an operator or user sees + rendered — admin/web/portal UI, Slack surfaces, emails — ships with a way for a + reviewer to see the result without booting it. Prefer a link to a live demo app + (e.g. the built UI served against a small mock API, published internally) so the + reviewer can click around the real thing; note in the PR what's mocked. Fall back + to screenshots only when a live demo isn't practical (e.g. Slack surfaces, emails), + and then show the after state (before/after for changes to something that existed), + rendered against realistic data. ## Private forks diff --git a/cli/README.md b/cli/README.md index 99bb94af6..d05635ea2 100644 --- a/cli/README.md +++ b/cli/README.md @@ -70,14 +70,11 @@ so it prints that snapshot as the matching data restore point (`aws rds restore-db-instance-from-db-snapshot`). Pre-deploy snapshots are pruned to a bounded count; `aws.predeployDbSnapshot: false` opts out. -`sandbox build` is a local validation build. `sandbox publish` pushes through the -configured OCI registry, resolves the image and base digests, records the base pin in -the config and the image pin in the config (docker/fly) or the durable AWS deployment -manifest, syncs the durable deployment layer when core is reachable, and repoints a -running Fly or AWS core. On AWS it requires `sandbox.backend: "sprites"` and, before -building anything, an existing deployment manifest and no `sandbox.image` override — -that override only seeds the first `qm up` and must be removed afterwards. Every -ordinary `up` also syncs the layer. +`sandbox build` and `sandbox publish` build and pin the Docker target's OCI sandbox +image. AWS uses `infra build-image` for its Lambda MicroVM guest. Fly uses the stock +Sprites runtime exposed by the installed SDK: custom sandbox images, Dockerfiles, +deployment-layer tools, and resident environment are rejected. Text skills remain +supported through the durable deployment layer. Auto uses its built-in model classifier unless `qm.config.jsonc` declares one `securityScreen` proxy with a provider label, HTTPS endpoint, and `shadow` or @@ -106,6 +103,8 @@ sandbox build [--from image] [--tag tag] [--dry-run] sandbox publish [--from image] [--app registry/repo] [--tag tag] [--dry-run] ``` +Rollback is available on AWS. The sandbox image commands apply to Docker. + All deploy commands accept `--config`, `--env-file`, and `--sandbox-dir`. `dev` remains the contributor worktree loop and is separate from the portable deployment contract. diff --git a/cli/package-lock.json b/cli/package-lock.json index 8b469f6a2..471eabca4 100644 --- a/cli/package-lock.json +++ b/cli/package-lock.json @@ -1,12 +1,12 @@ { "name": "@yc-software/qm", - "version": "0.1.6", + "version": "0.1.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@yc-software/qm", - "version": "0.1.6", + "version": "0.1.7", "license": "MIT", "bin": { "qm": "dist/bin/qm.js" diff --git a/cli/package.json b/cli/package.json index 4fbc357d4..588271b14 100644 --- a/cli/package.json +++ b/cli/package.json @@ -1,6 +1,6 @@ { "name": "@yc-software/qm", - "version": "0.1.6", + "version": "0.1.7", "license": "MIT", "description": "Control-plane CLI for portable QM deployments on Docker, Fly, and AWS.", "type": "module", diff --git a/cli/src/backends/doctor.ts b/cli/src/backends/doctor.ts index 37d1a204f..c466c6b3f 100644 --- a/cli/src/backends/doctor.ts +++ b/cli/src/backends/doctor.ts @@ -145,6 +145,8 @@ export async function doctorCommon( } if (config.target === "aws") { step("AWS Lambda MicroVM sandbox: configured"); + } else if (config.target === "fly") { + step("Fly Sprites stock sandbox: configured"); } else if (config.sandbox?.app) { requireFlyAuth(); try { diff --git a/cli/src/backends/fly.ts b/cli/src/backends/fly.ts index 06d9580a4..931748368 100644 --- a/cli/src/backends/fly.ts +++ b/cli/src/backends/fly.ts @@ -1,7 +1,7 @@ import { execFileSync, spawn, spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { dirname, join, resolve } from "node:path"; +import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { CliError, bold, die, dim, errMessage, header, note, ok, step, warn } from "../log.ts"; import { @@ -36,7 +36,6 @@ import { } from "../config.ts"; import { discoverPlugins, type ResolvedPlugin } from "../plugins.ts"; import { computedSecrets, runtimeSecretNames, secretDestinations, secretsForService } from "../secrets.ts"; -import { flySandboxRepository, imageRepository, pinnedByDigest, recordSandboxPin } from "../commands/sandbox.ts"; import { manifestRef } from "../manifest.ts"; import { doctorCommon, localDoctorSecrets, requireFlyAuth } from "./doctor.ts"; @@ -192,13 +191,7 @@ function deriveToml(ctx: FlyCtx, service: ServiceName): string { delete configuredEnv.FLY_ORG; delete configuredEnv.FLY_DEPLOY_BASE_IMAGE; } - const deploymentEnv = - service === "core" - ? { - ...(ctx.flyOrg ? { FLY_ORG: ctx.flyOrg } : {}), - ...(sandboxEnv.FLY_BASE_IMAGE ? { FLY_DEPLOY_BASE_IMAGE: sandboxEnv.FLY_BASE_IMAGE } : {}), - } - : {}; + const deploymentEnv: Record = service === "core" && ctx.flyOrg ? { FLY_ORG: ctx.flyOrg } : {}; const overrides: Record = { ...spec.managed(ctx.serviceCtx), ...sandboxEnv, @@ -454,16 +447,6 @@ function ensureApp(app: string, flyOrg: string, orgId: string, appPrefix: string note(`app ${app}: created`); } -function assertOwnedApp(app: string, flyOrg: string, orgId: string, appPrefix: string): void { - if (!flyOrgApps(flyOrg).has(app)) { - throw new CliError(`app ${app} is not present in configured Fly organization ${flyOrg}`); - } - const marker = flyOwnershipMarker(flyOrg, orgId, appPrefix); - if (!secretNames(app)?.has(marker)) { - throw new CliError(`app ${app} is not marked as owned by deployment ${flyDeploymentId(flyOrg, orgId, appPrefix)}`); - } -} - function ensurePostgres(ctx: FlyCtx): void { const app = `${ctx.appPrefix}-core`; const hasDatabaseUrl = secretNames(app)?.has("DATABASE_URL"); @@ -1317,67 +1300,12 @@ export function flyDown(config: QmConfig, configDir: string): void { ok("down — all apps scaled to 0."); } -export function flyPinSandbox(config: QmConfig, image: string, configDir = process.cwd()): void { - const appPrefix = appPrefixOf(config); - const app = `${appPrefix}-core`; - const flyOrg = config.flyOrg ?? ""; - if (!flyOrgApps(flyOrg).has(app)) { - note( - `${app} is not deployed in Fly organization ${flyOrg} — no live core to roll; the pin only takes effect from the config's sandbox.image on the next \`qm up\``, - ); - return; - } - assertOwnedApp(app, flyOrg, config.orgId, appPrefix); - let running: string; - try { - running = currentImage(app); - } catch { - note( - `${app} is not running — no live core to roll; the pin only takes effect from the config's sandbox.image on the next \`qm up\``, - ); - return; - } - const pinned: QmConfig = { ...config, sandbox: { ...config.sandbox, image } }; - const cfgPath = writeDerived(buildCtx(pinned, configDir, {}), "core"); - if (secretNames(app)?.has("FLY_BASE_IMAGE")) { - fly(["secrets", "unset", "--stage", "-a", app, "FLY_BASE_IMAGE"]); - note(`removed the stale FLY_BASE_IMAGE secret on ${app}; the derived [env] pin is authoritative`); - } - fly(["deploy", "--yes", "-c", cfgPath, "--image", running, ...serviceDef("core").fly!.deployFlags]); - ok(`${app} now boots sandboxes from ${image}`); +export function flyPinSandbox(_config: QmConfig, _image: string, _configDir = process.cwd()): void { + throw new CliError("Fly Sprites use the stock runtime and do not support sandbox image pins"); } -export function flyRollback(config: QmConfig, configPath: string, to?: string): void { - if (!to) throw new CliError("Fly rollback requires --to "); - if (to.startsWith("sha256:") && !/^sha256:[a-f0-9]{64}$/.test(to)) { - throw new CliError(`rollback --to must resolve to an image tag or sha256 digest (got ${JSON.stringify(to)})`); - } - let image: string; - if (to.includes("/")) { - image = to; - } else { - let repository: string | undefined; - if (config.sandbox?.image) repository = imageRepository(config.sandbox.image); - else if (config.sandbox?.app) repository = flySandboxRepository(config.sandbox.app); - if (!repository) { - throw new CliError( - "rollback cannot derive an image repository: the config has no sandbox.app or sandbox.image — " + - "pass a full ref instead (--to )", - ); - } - image = to.startsWith("sha256:") ? `${repository}@${to}` : `${repository}:${to}`; - } - const digestRef = /^\S+@sha256:[a-f0-9]{64}$/; - const slash = image.lastIndexOf("/"); - const colon = image.lastIndexOf(":"); - const taggedRef = colon > slash && /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(image.slice(colon + 1)); - if (image.includes("@") ? !digestRef.test(image) : !taggedRef) { - throw new CliError(`rollback --to must resolve to an image tag or sha256 digest (got ${JSON.stringify(image)})`); - } - const pinned = pinnedByDigest(image); - recordSandboxPin(configPath, pinned); - note(`recorded sandbox.image = ${pinned} in ${configPath}`); - flyPinSandbox(config, pinned, dirname(configPath)); +export function flyRollback(_config: QmConfig, _configPath: string, _to?: string): void { + throw new CliError("Fly rollback is unavailable because Sprites do not support sandbox image pins"); } export async function flyDoctor(config: QmConfig, configDir: string, envFile?: string): Promise { @@ -1437,9 +1365,6 @@ export function verifyLocalFlyTokens(config: QmConfig, secrets: ReadonlyMap publishFlySandbox(ctx, opts, true), + requiresSandboxApp: false, + publishSandbox: async () => { + throw new CliError( + "Fly Sprites use the stock runtime; the installed SDK cannot materialize images from `qm sandbox publish`", + ); + }, validateConfig: (config) => { const errors: Array<{ clause: string; message: string }> = [...sandboxImagePinErrors(config)]; if (!config.region?.trim()) @@ -293,19 +296,10 @@ const aws: HostingProvider = { coordinates: (config) => config.aws ? { accountOrOrganization: config.aws.accountId, region: config.aws.region } : {}, requiresSandboxApp: false, - publishSandbox: async (ctx, opts) => { - if (ctx.config.sandbox?.backend !== "sprites") { - throw new CliError( - `this AWS deployment runs Lambda MicroVM sandboxes (sandbox.backend is not "sprites"); use \`qm sandbox build\` to validate the layer and \`qm infra build-image\` to publish the runtime — or set "sandbox.backend": "sprites" with "sandbox.app" to host sandboxes in an operator-published layer image`, - ); - } - if (!opts.dryRun) assertAwsSandboxPinRecordable(ctx.config); - const published = runSandboxPublish(opts); - if (!published || opts.dryRun) return; - const config = loadConfigAt(ctx.configPath, { target: ctx.target }).config; - await hostingProvider(ctx.target) - .createBackend({ ...ctx, config }) - .pinSandbox(published.image); + publishSandbox: async () => { + throw new CliError( + 'AWS Lambda MicroVM sandboxes use `qm infra build-image`; "sandbox.backend": "sprites" custom images are unsupported', + ); }, validateConfig: (config, plugins) => { if (!config.aws) return []; diff --git a/cli/src/cli.ts b/cli/src/cli.ts index 148761850..282b8c928 100644 --- a/cli/src/cli.ts +++ b/cli/src/cli.ts @@ -146,12 +146,11 @@ ${bold("DEPLOY (operator)")} ${dim("— runs in the deployment directory")} status show what's running logs [] [-f] [--tail ] tail service logs (omit for all, interleaved) down [--purge] stop the deployment (--purge drops docker volumes) - rollback [--to ] roll back workloads (AWS: prior deployment manifest, - or manifest id/release label; Fly: sandbox image/tag) + rollback [--to ] roll back an AWS deployment manifest by id or release label sandbox build [--from ] [--tag ] [--dry-run] - build and validate the sandbox image locally + build and validate the Docker target's sandbox image locally sandbox publish [--from ] [--app ] [--tag ] [--dry-run] - build, push, resolve digest, and record the immutable pin + publish and pin the Docker target's sandbox image ${dim("Options (apply to all deploy commands):")} --config path to deploy config (default: qm.config.jsonc in deploy dir) diff --git a/cli/src/commands/check.ts b/cli/src/commands/check.ts index c1b23cff6..6562d9618 100644 --- a/cli/src/commands/check.ts +++ b/cli/src/commands/check.ts @@ -24,14 +24,14 @@ export function runChecks( opts: { report?: boolean } = {}, ): ChecksResult { const report = opts.report ?? true; - const layer = validateSandboxLayer(sandboxDir); + const layer = validateSandboxLayer(sandboxDir, config); const { plugins, errors: pluginErrors } = discoverPlugins(configDir, config); const configErrors: Array<{ clause: string; message: string }> = []; const configError = (message: string, clause = "config.v1"): void => void configErrors.push({ clause, message }); const provider = hostingProvider(config.target); configErrors.push(...provider.validateConfig(config, plugins)); if (provider.requiresSandboxApp && !config.sandbox?.app?.trim()) { - configError("contract sandbox.app: a Fly agent-computer app is required for docker and fly targets"); + configError("contract sandbox.app: a sandbox image registry app is required for the docker target"); } for (const skill of config.skills) { const path = resolve(configDir, skill); diff --git a/cli/src/commands/init.ts b/cli/src/commands/init.ts index 4572fa8f4..3e1c58b10 100644 --- a/cli/src/commands/init.ts +++ b/cli/src/commands/init.ts @@ -49,18 +49,20 @@ the scaffolded \`.gitignore\`. ## Customizing the sandbox -\`sandbox/\` defines what the agent gets in its execution environment: +\`sandbox/\` defines supported additions to the agent's execution environment: - A skill is \`sandbox/skills//SKILL.md\`: markdown with \`name\` and \`description\` frontmatter that teaches the agent a workflow and when to use it. -- A tool is \`sandbox/tools//tool.json\`: a descriptor whose minimal form is +- On image-customizable targets, a tool is \`sandbox/tools//tool.json\`: a descriptor whose minimal form is \`{ "id": ..., "advertise": ..., "install": { "binary": ... } }\`, with the executable next to it when the binary is not already in the base image. -- \`sandbox/Dockerfile\` is optional and only needed for system packages or +- On those targets, \`sandbox/Dockerfile\` is optional and only needed for system packages or runtimes. -The scaffold ships a working example, the \`greet\` skill and \`example-tool\`. -Copy its shape, then replace or delete it. +Fly uses the stock Sprites runtime exposed by the installed SDK, so it accepts +text skills and rejects tool descriptors, binaries, and Dockerfiles. The +scaffold always ships the \`greet\` skill; image-customizable targets also ship +\`example-tool\`. Copy the supported examples, then replace or delete them. ## The workflow @@ -94,6 +96,13 @@ description: Greet a teammate by name. Use whenever asked to say hello to someon Run \`example-tool \` to greet someone, e.g. \`example-tool Ada\`. `; +const SPRITES_GREET_SKILL = `--- +name: greet +description: Greet a teammate by name. Use whenever asked to say hello to someone. +--- +Reply with a friendly greeting addressed to the requested person. +`; + const EXAMPLE_TOOL_DESCRIPTOR = JSON.stringify({ id: "example-tool", advertise: "example-tool", install: { binary: "example-tool" } }, null, 2) + "\n"; @@ -116,8 +125,9 @@ function writeIfAbsent(dir: string, segments: string[], content: string, mode?: ok(`wrote ${rel}`); } -function scaffoldSandbox(dir: string): void { - writeIfAbsent(dir, ["sandbox", "skills", "greet", "SKILL.md"], GREET_SKILL); +function scaffoldSandbox(dir: string, target: Target): void { + writeIfAbsent(dir, ["sandbox", "skills", "greet", "SKILL.md"], target === "fly" ? SPRITES_GREET_SKILL : GREET_SKILL); + if (target === "fly") return; writeIfAbsent(dir, ["sandbox", "tools", "example-tool", "tool.json"], EXAMPLE_TOOL_DESCRIPTOR); writeIfAbsent(dir, ["sandbox", "tools", "example-tool", "example-tool"], EXAMPLE_TOOL_BIN, 0o755); } @@ -295,7 +305,7 @@ export function runInit(opts: { const manifests = renderSlackManifests(config); writeIfAbsent(dir, ["slack-app-manifest.yml"], manifests.bot); if (usesSlackOidc(config)) writeIfAbsent(dir, ["slack-sso-manifest.yml"], manifests.sso); - scaffoldSandbox(dir); + scaffoldSandbox(dir, target); for (const file of provider.scaffold.files(config)) writeIfAbsent(dir, file.segments, file.content); note(""); diff --git a/cli/src/commands/sandbox.ts b/cli/src/commands/sandbox.ts index fb5cd512f..42fb96194 100644 --- a/cli/src/commands/sandbox.ts +++ b/cli/src/commands/sandbox.ts @@ -248,8 +248,13 @@ function assertPublishPlatform(body: string): void { } function prepare(opts: SandboxBuildOpts): PreparedBuild { + if (opts.config.target === "fly") { + throw new CliError( + "Fly Sprites use the stock runtime; the installed SDK cannot materialize images built by `qm sandbox build` or `qm sandbox publish`", + ); + } const sandboxDir = resolve(opts.sandboxDir); - const layer = validateSandboxLayer(sandboxDir); + const layer = validateSandboxLayer(sandboxDir, opts.config); if (layer.errors.length) { throw new CliError(`sandbox check failed:\n${layer.errors.map((error) => ` - ${error}`).join("\n")}`); } diff --git a/cli/src/config.ts b/cli/src/config.ts index 2277e65cf..a6e8db898 100644 --- a/cli/src/config.ts +++ b/cli/src/config.ts @@ -198,7 +198,7 @@ export const isDigestPinned = (ref: string): boolean => /@sha256:[0-9a-f]{64}$/. const SANDBOX_PIN_PENDING = `"sandbox.app" is set but no sandbox layer image is pinned; run \`qm sandbox publish\` to build and record the digest-pinned "sandbox.image" agents boot from`; export const sandboxPinPending = (config: QmConfig): boolean => - config.target !== "aws" && Boolean(config.sandbox?.app && !config.sandbox.image); + config.target === "docker" && Boolean(config.sandbox?.app && !config.sandbox.image); export function sandboxImagePinErrors(config: QmConfig): Array<{ clause: string; message: string }> { const sb = config.sandbox; @@ -218,6 +218,7 @@ export function sandboxCoreEnv( const env: Record = {}; const missingSecrets: string[] = []; const sb = config.sandbox; + if (config.target === "fly") return { env: { SANDBOX_BACKEND: "sprites" }, missingSecrets }; if (!sb) return { env, missingSecrets }; if (sb.app) { if (!sb.image) throw new CliError(SANDBOX_PIN_PENDING, { clause: "config.v1" }); @@ -225,8 +226,6 @@ export function sandboxCoreEnv( if (violation) throw new CliError(violation.message, { clause: violation.clause }); env.FLY_SANDBOX_APP_NAME = sb.app; env.FLY_BASE_IMAGE = sb.image; - const backend = sb.backend ?? (config.target === "fly" ? "sprites" : undefined); - if (backend) env.SANDBOX_BACKEND = backend; } for (const [k, v] of Object.entries(sb.env ?? {})) env[`FLY_RESIDENT_ENV_${k}`] = v; for (const name of sb.secretEnv ?? []) { @@ -633,6 +632,21 @@ function validate(raw: unknown, path: string): QmConfig { return v; }); const sandbox = validateSandbox(o["sandbox"], path, target); + if (target === "fly") { + const unsupportedRuntimeEnv = (name: string): boolean => name === "SANDBOX_BACKEND" || name.startsWith("FLY_"); + const configured = Object.keys(env.core ?? {}).find(unsupportedRuntimeEnv); + if (configured) { + throw new CliError( + `${path}: Fly Sprites use the stock runtime and do not support "env.core.${configured}"; remove it`, + ); + } + const routed = Object.keys(secretEnv.core ?? {}).find(unsupportedRuntimeEnv); + if (routed) { + throw new CliError( + `${path}: Fly Sprites use the stock runtime and do not support "secretEnv.core.${routed}"; remove it`, + ); + } + } const out: QmConfig = { contract, @@ -1254,14 +1268,14 @@ function validateSandbox(raw: unknown, path: string, target: Target): SandboxCon if (o["backend"] !== undefined) { if (o["backend"] !== "sprites" && o["backend"] !== "aws") { throw new CliError( - `${path}: "sandbox.backend" must be "sprites" (Fly Sprites, booting the operator-published layer image from the Fly app in "sandbox.app") or "aws" (Lambda MicroVM sandboxes)`, + `${path}: "sandbox.backend" must be "sprites" (the stock Fly Sprites runtime) or "aws" (Lambda MicroVM sandboxes)`, ); } out.backend = o["backend"]; } if (o["app"] !== undefined) { if (typeof o["app"] !== "string" || !o["app"].trim()) { - throw new CliError(`${path}: "sandbox.app" must be a non-empty string (the Fly sandbox app name)`); + throw new CliError(`${path}: "sandbox.app" must be a non-empty string (the Docker sandbox image repository)`); } out.app = o["app"]; } @@ -1296,21 +1310,31 @@ function validateSandbox(raw: unknown, path: string, target: Target): SandboxCon const stray = (["app", "image", "baseImage", "env", "secretEnv"] as const).filter((key) => out[key] !== undefined); if (stray.length) { throw new CliError( - `${path}: "sandbox.backend": "aws" runs Lambda MicroVM sandboxes, which ignore ${stray.map((key) => `"sandbox.${key}"`).join(", ")} (Fly layer-image settings) — remove them or set "sandbox.backend": "sprites"`, + `${path}: "sandbox.backend": "aws" runs Lambda MicroVM sandboxes, which ignore ${stray.map((key) => `"sandbox.${key}"`).join(", ")} — remove them`, ); } } - if (out.image && !out.app) { - throw new CliError(`${path}: "sandbox.image" requires "sandbox.app" (the app the microVMs run in)`); - } - if (out.backend === "sprites" && !out.app) { + if (out.backend === "sprites" && target !== "fly") { throw new CliError( - `${path}: "sandbox.backend": ${JSON.stringify(out.backend)} requires "sandbox.app" (the Fly app agents execute in)`, + `${path}: "sandbox.backend": "sprites" is only supported for target "fly"; the installed Sprites SDK cannot materialize the custom image contract previously used on other targets`, + ); + } + if (target === "fly") { + const unsupported = (["app", "image", "baseImage", "env", "secretEnv"] as const).filter( + (key) => out[key] !== undefined, ); + if (unsupported.length) { + throw new CliError( + `${path}: Fly Sprites use the stock runtime and do not support ${unsupported.map((key) => `"sandbox.${key}"`).join(", ")}; remove ${unsupported.length === 1 ? "it" : "them"}`, + ); + } + } + if (out.image && !out.app) { + throw new CliError(`${path}: "sandbox.image" requires "sandbox.app" (the image repository)`); } if (target === "aws" && out.backend === undefined) { throw new CliError( - `${path}: target "aws" requires an explicit "sandbox.backend" — "sprites" boots the operator-published layer image in "sandbox.app"; "aws" runs Lambda MicroVM sandboxes (or omit the whole "sandbox" block for the MicroVM default)`, + `${path}: target "aws" requires "sandbox.backend": "aws", or omit the whole "sandbox" block for the Lambda MicroVM default`, ); } return out; diff --git a/cli/src/provider-scaffold.ts b/cli/src/provider-scaffold.ts index 408995896..ab7ad3424 100644 --- a/cli/src/provider-scaffold.ts +++ b/cli/src/provider-scaffold.ts @@ -156,15 +156,12 @@ export const dockerScaffold: ProviderScaffold = { env: `{ "core": { "HARNESS": "pi" } }`, secretEnv: "", sandbox: `, - - // The Fly app agents execute in. The core boots the immutable sandbox image - // recorded by \`qm sandbox publish\`. "sandbox": { "app": ${JSON.stringify(`${orgId}-sandboxes`)} }`, }), ignores: [".env", "node_modules/", ".generated/"], agentsAppendix: "", files: noFiles, - configurationHint: "docker: confirm the local public port and Fly sandbox app before setup", + configurationHint: "docker: confirm the local public port and sandbox image registry before setup", finalCommand: "npm exec qm -- up", finalWhy: "pull images, start services, print URLs", }; @@ -189,11 +186,7 @@ export const flyScaffold: ProviderScaffold = { // The initial admin seed is kept in the provider secret store, never in config. "secretEnv": { "core": { "ADMIN_GRANTS": "ADMIN_GRANTS" } },`, - sandbox: ` - - // The Fly app agents execute in. The core boots the immutable sandbox image - // recorded by \`qm sandbox publish\`. - "sandbox": { "app": ${JSON.stringify(`${orgId}-sandboxes`)} }`, + sandbox: "", }), ignores: [".env", "node_modules/", ".generated/"], agentsAppendix: "", @@ -247,12 +240,7 @@ export const awsScaffold: ProviderScaffold = { // The initial admin seed is kept in the provider secret store, never in config. "secretEnv": { "core": { "ADMIN_GRANTS": "ADMIN_GRANTS" } }`, - sandbox: ` - - // Where agent sandboxes execute. Omitting "sandbox" entirely runs AWS Lambda MicroVMs - // (published by \`qm infra build-image\`). To boot an operator-published sandbox layer - // image in a Fly app instead (published by \`qm sandbox publish\`), declare it explicitly: - // "sandbox": { "backend": "sprites", "app": ${JSON.stringify(`${orgId}-sandboxes`)} }`, + sandbox: "", }); }, ignores: [ diff --git a/cli/src/sandbox-layer.ts b/cli/src/sandbox-layer.ts index 49d184cce..a2addd33f 100644 --- a/cli/src/sandbox-layer.ts +++ b/cli/src/sandbox-layer.ts @@ -1,6 +1,7 @@ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import { join } from "node:path"; import { JUNK_FILE, deploymentLayerBundle } from "./deployment-layer.ts"; +import type { QmConfig } from "./config.ts"; import { errMessage } from "./log.ts"; export type ApprovalDecision = "require_approval" | "deny"; @@ -787,7 +788,8 @@ const subdirs = (dir: string): string[] => { const isCidr = (host: string): boolean => /^\d{1,3}(\.\d{1,3}){3}\/\d{1,2}$/.test(host) || /^[0-9A-Fa-f:]+\/\d{1,3}$/.test(host); -export function validateSandboxLayer(sandboxDir: string): SandboxValidation { +export function validateSandboxLayer(sandboxDir: string, config?: QmConfig): SandboxValidation { + const sprites = config?.target === "fly"; const out: SandboxValidation = { exists: existsSync(sandboxDir), hasDockerfile: existsSync(join(sandboxDir, "Dockerfile")), @@ -835,7 +837,11 @@ export function validateSandboxLayer(sandboxDir: string): SandboxValidation { const binaryName = descriptor.install?.binary ?? descriptor.id; const exePath = join(toolsDir, name, binaryName); const hasExe = isFile(exePath); - if (!hasExe && !out.hasDockerfile) { + if (sprites) { + out.errors.push( + `tool "${descriptor.id}" cannot be installed in Fly Sprites because the installed SDK cannot materialize tool binaries or a custom image; remove it`, + ); + } else if (!hasExe && !out.hasDockerfile) { out.errors.push( `tool "${descriptor.id}" (tools/${name}/) can't get its binary on PATH: ship an executable ` + `"${binaryName}" in the folder, or add a sandbox/Dockerfile that installs it`, @@ -899,5 +905,11 @@ export function validateSandboxLayer(sandboxDir: string): SandboxValidation { } } + if (sprites && out.hasDockerfile) { + out.errors.push( + "sandbox/Dockerfile cannot be applied to Fly Sprites because the installed SDK cannot materialize a custom image; remove it", + ); + } + return out; } diff --git a/cli/src/secrets.ts b/cli/src/secrets.ts index ed061840b..1242ee749 100644 --- a/cli/src/secrets.ts +++ b/cli/src/secrets.ts @@ -109,14 +109,6 @@ export const FIRST_PARTY_SECRET_SPECS: readonly SecretSpec[] = [ description: "Stable signing key for reviewed skills.", generate: MINT_LOCALLY, }, - { - name: "FLY_SANDBOX_API_TOKEN", - service: "core", - envName: "FLY_API_TOKEN", - required: { when: { kind: "target", target: "fly" } }, - description: "Fly deploy token scoped to the agent-computer app.", - generate: "fly tokens create deploy -a -x 8760h", - }, { name: "FLY_DEPLOY_API_TOKEN", service: "core", diff --git a/cli/templates/deployment/references/fly.md b/cli/templates/deployment/references/fly.md index 422fc9a40..aafdac2df 100644 --- a/cli/templates/deployment/references/fly.md +++ b/cli/templates/deployment/references/fly.md @@ -4,40 +4,29 @@ Use this after the choices and billing confirmation in `deployment.md`. ## Preflight -Require authenticated Flyctl, Docker Buildx, and permission to create apps, -Managed Postgres, and private object storage: +Require authenticated Flyctl and permission to create apps, Managed Postgres, +and private object storage: ```bash fly auth whoami fly orgs list -docker buildx version ``` -Set `flyOrg`, `region`, globally unique `appPrefix`, `publicUrl`, and -`sandbox.app`. The public origin is normally -`https://-portal.fly.dev`. Confirm the service and sandbox app names -are available in the selected organization. - -Create the sandbox registry app before setup. Setup refuses to mint a token -until the app exists and verifies that the new app-scoped token can list its -Machines: +Set `flyOrg`, `region`, globally unique `appPrefix`, and `publicUrl`. The +public origin is normally `https://-portal.fly.dev`. Confirm the +service app names are available in the selected organization. Fly agent +computers use the stock Sprites runtime; do not configure a sandbox app, image, +resident environment, Dockerfile, or tool binaries. ```bash -fly apps create --org npm exec qm -- setup . npm exec qm -- slack render npm exec qm -- check ``` -Setup keeps the sandbox deploy token separate from the organization deploy -token. Do not substitute a personal token. - -## Publish the agent computer and deploy - -Publish the package-selected sandbox base and record its immutable digest: +## Deploy ```bash -npm exec qm -- sandbox publish npm exec qm -- secrets push npm exec qm -- plan npm exec qm -- up @@ -55,17 +44,16 @@ reconciles the same apps. ## Agent-computer proof -Use the exact signed-in principal to select one sandbox Machine by its -`agent_scope` metadata. Read `/root/workspace/qm-computer-proof.txt` with -`fly machine exec` and require it to match the UUID created in the browser. A -missing or ambiguous scope match is a failed proof. +As the exact signed-in principal, ask the agent to write a fresh UUID to +`/home/sprite/workspace/qm-computer-proof.txt`. Start a later turn in the same +scope, ask it to read the file, and require the value to match. A missing or +different value is a failed persistence proof. Routine operations: ```bash npm exec qm -- status npm exec qm -- logs core --follow -npm exec qm -- rollback --to npm exec qm -- down ``` diff --git a/cli/templates/fly/core.toml b/cli/templates/fly/core.toml index 386a6ec02..f662cc7a7 100644 --- a/cli/templates/fly/core.toml +++ b/cli/templates/fly/core.toml @@ -15,12 +15,9 @@ primary_region = "sjc" S3_REGION = "auto" DATA_DIR = "/data" PUBLIC_WEB_URL = "https://agent.example.com" - FLY_SANDBOX_APP_NAME = "acme-sandboxes" - FLY_BASE_IMAGE = "registry.fly.io/acme-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a" WEB_UI_PUBLIC_URL = "https://agent.example.com" REQUIRE_SIGNED_PORTAL_IDENTITY = "1" FLY_ORG = "personal" - FLY_DEPLOY_BASE_IMAGE = "registry.fly.io/acme-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a" FLY_DEPLOY_APP_PREFIX = "qm-d" [deploy] strategy = "bluegreen" diff --git a/cli/test/check.test.ts b/cli/test/check.test.ts index 5c1d504f8..fec39a1a1 100644 --- a/cli/test/check.test.ts +++ b/cli/test/check.test.ts @@ -134,6 +134,40 @@ test("a tool with no executable BUT a sandbox/Dockerfile passes (Dockerfile inst } }); +test("Fly Sprites accept skills and reject every sandbox image tool path", () => { + const flyConfig: Partial = { + target: "fly", + publicUrl: "https://acme.example.com", + region: "sjc", + flyOrg: "personal", + env: { core: { SNAPSHOT_STORE: "s3", TRANSFER_STORE: "s3", S3_BUCKET: "acme-data", S3_REGION: "auto" } }, + sandbox: undefined, + }; + const customized = deployment((dir) => { + writeTool(dir, "custom-tool", { id: "custom-tool", install: { binary: "custom-tool" } }); + writeFileSync(join(dir, "sandbox", "Dockerfile"), "FROM base\n"); + writeSkill(dir, "greet", "name: greet\ndescription: Greet a teammate."); + }, flyConfig); + const skillOnly = deployment( + (dir) => writeSkill(dir, "greet", "name: greet\ndescription: Greet a teammate."), + flyConfig, + ); + try { + assert.throws( + () => check(customized), + (error) => { + assert.match(String(error), /tool "custom-tool" cannot be installed in Fly Sprites/); + assert.match(String(error), /sandbox\/Dockerfile cannot be applied to Fly Sprites/); + return true; + }, + ); + assert.doesNotThrow(() => check(skillOnly)); + } finally { + rmSync(customized.dir, { recursive: true, force: true }); + rmSync(skillOnly.dir, { recursive: true, force: true }); + } +}); + test("duplicate tool ids are flagged", () => { const d = deployment((dir) => { writeTool(dir, "folderA", { id: "same", install: { binary: "same" } }); diff --git a/cli/test/cli-dispatch.test.ts b/cli/test/cli-dispatch.test.ts index d5b2bf131..7774d3790 100644 --- a/cli/test/cli-dispatch.test.ts +++ b/cli/test/cli-dispatch.test.ts @@ -200,7 +200,7 @@ else console.log("{}"); services: ["core"], env: { core: { AWS_DEPLOY_IMAGE: "acme-microvm-app", AWS_DEPLOY_IMAGE_VERSION: "1" } }, imageOverrides: { core: `ghcr.io/acme/core@sha256:${"a".repeat(64)}` }, - sandbox: { backend: "sprites", app: "acme-sandboxes", image: PINNED_SANDBOX_IMAGE }, + sandbox: { backend: "aws" }, aws: { accountId: "123456789012", region: "us-west-2", @@ -242,7 +242,7 @@ test("successful check --json --live reports the live-drift clause", async () => services: ["core"], env: { core: { AWS_DEPLOY_IMAGE: "acme-microvm-app", AWS_DEPLOY_IMAGE_VERSION: "1" } }, imageOverrides: { core: `ghcr.io/acme/core@${digest}` }, - sandbox: { backend: "sprites", app: "acme-sandboxes", image: PINNED_SANDBOX_IMAGE }, + sandbox: { backend: "aws" }, aws: { accountId: "123456789012", region: "us-west-2", @@ -447,7 +447,7 @@ test("sandbox publish directs MicroVM AWS deployments (no sandbox.app) to the im } }); -test("sandbox publish on an AWS deployment with sandbox.app dry-runs the operator layer image", async () => { +test("sandbox publish rejects the removed AWS Sprites image mode at config load", async () => { const dir = mkdtempSync(join(tmpdir(), "qm-dispatch-")); const configPath = join(dir, CONFIG_FILENAME); const raw = JSON.stringify({ @@ -474,9 +474,8 @@ test("sandbox publish on an AWS deployment with sandbox.app dry-runs the operato writeFileSync(join(dir, "sandbox", "Dockerfile"), "FROM scratch\n"); try { const result = await run(["sandbox", "publish", "--dry-run"], dir); - assert.equal(result.exitCode, null, result.out); - assert.match(result.out, /sandbox publish → registry\.fly\.io\/acme-sandboxes:latest/); - assert.match(result.out, /DRY RUN — nothing built, pushed, or recorded/); + assert.equal(result.exitCode, 1, result.out); + assert.match(result.out, /"sandbox.backend": "sprites" is only supported for target "fly"/); assert.equal(readFileSync(configPath, "utf8"), raw); } finally { rmSync(dir, { recursive: true, force: true }); @@ -638,7 +637,8 @@ test("config get prints raw scalars and JSON objects, honors --target, and fails ); const overridden = await run(["config", "get", "target", "--target", "fly"], dir); - assert.equal(overridden.out, "fly", "--target overrides the config's durable value, same as every deploy command"); + assert.equal(overridden.exitCode, 1); + assert.match(overridden.out, /Fly Sprites use the stock runtime and do not support/); const missing = await run(["config", "get", "aws.deployRoleArn"], dir); assert.equal(missing.exitCode, 1); @@ -662,7 +662,6 @@ test("--target revalidates the effective provider config", async () => { publicUrl: "http://localhost:8080", target: "docker", services: ["core"], - sandbox: { backend: "sprites", app: "acme-sandboxes", image: PINNED_SANDBOX_IMAGE }, }), ); try { diff --git a/cli/test/config.test.ts b/cli/test/config.test.ts index 7d62298de..f816d96a4 100644 --- a/cli/test/config.test.ts +++ b/cli/test/config.test.ts @@ -712,43 +712,37 @@ test("a tag-pinned sandbox.image is refused: staleness compares image references }); }); -test("a mutable sandbox tag fails check, while an unpublished deployment is only pending", () => { - withConfig({ target: "fly", region: "sjc", flyOrg: "acme", sandbox: { app: "acme-sandboxes" } }, ({ path }) => { +test("Fly Sprites reject every unsupported image and resident-environment setting", () => { + for (const sandbox of [ + { app: "acme-sandboxes" }, + { image: `registry.fly.io/acme-sandboxes@sha256:${"1a".repeat(32)}` }, + { baseImage: `registry.example/base@sha256:${"2b".repeat(32)}` }, + { env: { TZ: "UTC" } }, + { secretEnv: ["COMPANY_API_TOKEN"] }, + ]) { + withConfig({ target: "fly", region: "sjc", flyOrg: "acme", sandbox }, ({ path }) => { + assert.throws(() => loadConfigAt(path), /Fly Sprites use the stock runtime and do not support/); + }); + } + withConfig({ target: "fly", region: "sjc", flyOrg: "acme" }, ({ path }) => { const { config } = loadConfigAt(path); - assert.deepEqual(sandboxImagePinErrors(config), [], "check cannot demand a pin only `sandbox publish` can write"); - assert.equal(sandboxPinPending(config), true); - assert.throws( - () => sandboxCoreEnv(config), - /no sandbox layer image is pinned/, - "rendering core still fails closed", - ); + assert.equal(sandboxPinPending(config), false); + assert.deepEqual(sandboxImagePinErrors(config), []); + assert.deepEqual(sandboxCoreEnv(config), { env: { SANDBOX_BACKEND: "sprites" }, missingSecrets: [] }); }); - withConfig( - { - target: "fly", - region: "sjc", - flyOrg: "acme", - sandbox: { app: "acme-sandboxes", image: "registry.fly.io/acme-sandboxes:latest" }, - }, - ({ path }) => { - const { config } = loadConfigAt(path); - const errors = sandboxImagePinErrors(config); - assert.equal(errors.length, 1); - assert.equal(errors[0]!.clause, "config.v1"); - assert.match(errors[0]!.message, /must be pinned by digest/); - }, - ); - withConfig( - { - target: "fly", - region: "sjc", - flyOrg: "acme", - sandbox: { app: "acme-sandboxes", image: `registry.fly.io/acme-sandboxes@sha256:${"1a".repeat(32)}` }, - }, - ({ path }) => { - assert.deepEqual(sandboxImagePinErrors(loadConfigAt(path).config), []); - }, - ); + withConfig({ target: "fly", region: "sjc", flyOrg: "acme", sandbox: { backend: "sprites" } }, ({ path }) => { + assert.deepEqual(loadConfigAt(path).config.sandbox, { backend: "sprites" }); + }); + for (const override of [ + { env: { core: { FLY_BASE_IMAGE: "registry.example/acme:latest" } } }, + { env: { core: { FLY_CPUS: "99" } } }, + { env: { core: { SANDBOX_BACKEND: "aws" } } }, + { secretEnv: { core: { FLY_API_TOKEN: "TOKEN" } } }, + ]) { + withConfig({ target: "fly", region: "sjc", flyOrg: "acme", ...override }, ({ path }) => { + assert.throws(() => loadConfigAt(path), /Fly Sprites use the stock runtime and do not support/); + }); + } }); test("sandbox.image requires sandbox.app and must be non-empty", () => { @@ -812,7 +806,7 @@ test("sandbox shape errors: object, app non-empty string, env string-map, secret { sandbox: { secretEnv: ["1BAD"] }, rx: /not a valid env var name/ }, { sandbox: { backend: "k8s", app: "acme-sandboxes" }, rx: /"sandbox.backend" must be "sprites".*or "aws"/ }, { sandbox: { backend: "fly", app: "acme-sandboxes" }, rx: /"sandbox.backend" must be "sprites".*or "aws"/ }, - { sandbox: { backend: "sprites" }, rx: /"sandbox.backend": "sprites" requires "sandbox.app"/ }, + { sandbox: { backend: "sprites" }, rx: /"sandbox.backend": "sprites" is only supported for target "fly"/ }, { sandbox: { backend: "aws", app: "acme-sandboxes" }, rx: /"sandbox.backend": "aws" \(Lambda MicroVM sandboxes\) requires target "aws"/, @@ -825,7 +819,7 @@ test("sandbox shape errors: object, app non-empty string, env string-map, secret } }); -test("aws target makes the sandbox substrate explicit: backend required with a sandbox block, sprites needs app, aws forbids fly-image settings", () => { +test("aws target accepts only its MicroVM backend and rejects the old Sprites image mode", () => { const aws = { accountId: "123456789012", region: "us-west-2", @@ -837,17 +831,14 @@ test("aws target makes the sandbox substrate explicit: backend required with a s services: { core: { ecrRepository: "core", ecsService: "acme-core", cpu: 512, memory: 1024 } }, }; withConfig({ target: "aws", aws, sandbox: { app: "acme-sandboxes" } }, ({ path }) => { - assert.throws(() => loadConfigAt(path), /target "aws" requires an explicit "sandbox.backend"/); + assert.throws(() => loadConfigAt(path), /target "aws" requires "sandbox.backend": "aws"/); }); withConfig({ target: "aws", aws, sandbox: { backend: "sprites", app: "acme-sandboxes" } }, ({ path }) => { - assert.equal(loadConfigAt(path).config.sandbox?.backend, "sprites"); + assert.throws(() => loadConfigAt(path), /"sandbox.backend": "sprites" is only supported for target "fly"/); }); withConfig({ target: "aws", aws, sandbox: { backend: "aws" } }, ({ path }) => { assert.equal(loadConfigAt(path).config.sandbox?.backend, "aws"); }); - withConfig({ target: "aws", aws, sandbox: { backend: "sprites", app: "acme-sandboxes" } }, ({ path }) => { - assert.equal(loadConfigAt(path).config.sandbox?.backend, "sprites"); - }); withConfig({ target: "aws", aws, sandbox: { backend: "aws", app: "acme-sandboxes" } }, ({ path }) => { assert.throws( () => loadConfigAt(path), diff --git a/cli/test/doctor.test.ts b/cli/test/doctor.test.ts index b94130b0d..4e379015f 100644 --- a/cli/test/doctor.test.ts +++ b/cli/test/doctor.test.ts @@ -516,7 +516,7 @@ test("fly doctor reports a missing flyctl before trying `fly secrets list`", asy } }); -test("Fly doctor token probes reject expired scoped tokens without exposing them", () => { +test("Fly doctor token probes only the enabled deployment publisher token", () => { const dir = mkdtempSync(join(tmpdir(), "qm-doctor-fly-token-")); const bin = join(dir, "fake-fly.cjs"); writeFileSync( @@ -529,9 +529,7 @@ process.exit(1); ); chmodSync(bin, 0o755); const prior = process.env.FLY_BIN; - const priorSandboxToken = process.env.FLY_SANDBOX_API_TOKEN; process.env.FLY_BIN = bin; - process.env.FLY_SANDBOX_API_TOKEN = "FlyV1-good"; const flyConfig: QmConfig = { ...config, target: "fly", @@ -540,23 +538,8 @@ process.exit(1); flyOrg: "personal", }; try { - assert.throws( - () => verifyLocalFlyTokens(flyConfig, new Map([["FLY_SANDBOX_API_TOKEN", "FlyV1-expired"]])), - (error: unknown) => { - assert.match((error as Error).message, /FLY_SANDBOX_API_TOKEN was rejected/); - assert.doesNotMatch((error as Error).message, /FlyV1-expired/); - return true; - }, - ); - assert.doesNotThrow(() => - verifyLocalFlyTokens( - flyConfig, - new Map([ - ["FLY_SANDBOX_API_TOKEN", "FlyV1-good"], - ["FLY_DEPLOY_API_TOKEN", "FlyV1-expired"], - ]), - ), - ); + assert.doesNotThrow(() => verifyLocalFlyTokens(flyConfig, new Map([["FLY_SANDBOX_API_TOKEN", "FlyV1-expired"]]))); + assert.doesNotThrow(() => verifyLocalFlyTokens(flyConfig, new Map([["FLY_DEPLOY_API_TOKEN", "FlyV1-expired"]]))); assert.throws( () => verifyLocalFlyTokens( @@ -564,18 +547,17 @@ process.exit(1); ...flyConfig, env: { ...flyConfig.env, core: { ...flyConfig.env.core, DEPLOY_PROVIDER: "fly" } }, }, - new Map([ - ["FLY_SANDBOX_API_TOKEN", "FlyV1-good"], - ["FLY_DEPLOY_API_TOKEN", "FlyV1-expired"], - ]), + new Map([["FLY_DEPLOY_API_TOKEN", "FlyV1-expired"]]), ), - /FLY_DEPLOY_API_TOKEN was rejected/, + (error: unknown) => { + assert.match((error as Error).message, /FLY_DEPLOY_API_TOKEN was rejected/); + assert.doesNotMatch((error as Error).message, /FlyV1-expired/); + return true; + }, ); } finally { if (prior === undefined) delete process.env.FLY_BIN; else process.env.FLY_BIN = prior; - if (priorSandboxToken === undefined) delete process.env.FLY_SANDBOX_API_TOKEN; - else process.env.FLY_SANDBOX_API_TOKEN = priorSandboxToken; rmSync(dir, { recursive: true, force: true }); } }); diff --git a/cli/test/fixtures/imagefrom-stack.json b/cli/test/fixtures/imagefrom-stack.json index 832aaa36e..2c10bdff0 100644 --- a/cli/test/fixtures/imagefrom-stack.json +++ b/cli/test/fixtures/imagefrom-stack.json @@ -7,10 +7,6 @@ "region": "sjc", "flyOrg": "personal", "imageFrom": "qm", - "sandbox": { - "app": "beta-sandboxes", - "image": "registry.fly.io/beta-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a" - }, "services": ["core", "slack", "admin", "web-ui", "portal"], "vms": { "core": { "size": "shared-cpu-2x", "memory": "4gb" } @@ -18,7 +14,6 @@ "env": { "core": { "PUBLIC_WEB_URL": "https://beta-portal.fly.dev", - "FLY_SANDBOX_APP_NAME": "beta-sandboxes", "SNAPSHOT_STORE": "s3", "TRANSFER_STORE": "s3", "S3_BUCKET": "beta-data", diff --git a/cli/test/fly-derive.test.ts b/cli/test/fly-derive.test.ts index 21117ee97..ba64fc14a 100644 --- a/cli/test/fly-derive.test.ts +++ b/cli/test/fly-derive.test.ts @@ -37,7 +37,7 @@ test("an imageFrom stack reuses the reference images and overrides only its own assert.match(coreToml, /app = "beta-core"/); assert.match(coreToml, /ORG_ID = "beta"/); assert.match(coreToml, /PUBLIC_WEB_URL = "https:\/\/beta-portal\.fly\.dev"/); - assert.match(coreToml, /FLY_SANDBOX_APP_NAME = "beta-sandboxes"/); + assert.match(coreToml, /SANDBOX_BACKEND = "sprites"/); }); test("derived Fly configs contain only the deployment's region, org, sandbox, and portal policy", () => { @@ -50,11 +50,6 @@ test("derived Fly configs contain only the deployment's region, org, sandbox, an appPrefix: "example-stack", region: "ord", flyOrg: "example-org", - sandbox: { - app: "example-sandboxes", - image: - "registry.fly.io/example-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a", - }, services: ["core", "admin", "web-ui", "portal"], plugins: [], skills: [], @@ -70,10 +65,7 @@ test("derived Fly configs contain only the deployment's region, org, sandbox, an assert.match(admin, /^\s*ADMIN_BASE_PATH = "\/admin"$/m); assert.match(core, /^\s*FLY_ORG = "example-org"$/m); assert.match(core, /^\s*PI_MODEL = "example-model"$/m); - assert.match( - core, - /^\s*FLY_DEPLOY_BASE_IMAGE = "registry\.fly\.io\/example-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a"$/m, - ); + assert.match(core, /^\s*SANDBOX_BACKEND = "sprites"$/m); assert.match(core, /^\s*QM_DEPLOYMENT_ID = "qm-v2:example-org:example:example-stack"$/m); assert.doesNotMatch(core, /PI_DETECT_MODEL/); assert.doesNotMatch(portal, /OIDC_ALLOWED_EMAIL_DOMAIN/); diff --git a/cli/test/fly-sandbox.test.ts b/cli/test/fly-sandbox.test.ts index 104c71ec6..38d66504d 100644 --- a/cli/test/fly-sandbox.test.ts +++ b/cli/test/fly-sandbox.test.ts @@ -1,4 +1,3 @@ -import { createHash } from "node:crypto"; import { spawnSync } from "node:child_process"; import { test } from "node:test"; import assert from "node:assert/strict"; @@ -23,35 +22,14 @@ test("Fly runs the live session smoke inside the private core machine", () => { assert.equal(flyLiveSessionCommand(), "node src/deployment/postdeploy-smoke.ts session http://127.0.0.1:8080"); }); -test("a fly config's sandbox block rewrites the core fly.toml [env] (app, image, env literals)", () => { +test("a fly config uses the stock Sprites runtime", () => { const { config } = loadConfigAt(join(repoRoot, "deploy", "stacks", "acme", "qm.config.jsonc")); - const cfg = { - ...config, - sandbox: { - app: "acme-sb", - image: "registry.fly.io/acme-sb@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a", - env: { TZ: "UTC" }, - secretEnv: ["COMPANY_API_TOKEN"], - }, - }; - - const core = derivedTomlFor(cfg, "core", repoRoot); - assert.match(core, /FLY_SANDBOX_APP_NAME = "acme-sb"/, "sandbox.app overrides the baked app name"); - assert.match( - core, - /FLY_BASE_IMAGE = "registry\.fly\.io\/acme-sb@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a"/, - "FLY_BASE_IMAGE is the config's digest pin", - ); - assert.match(core, /FLY_RESIDENT_ENV_TZ = "UTC"/, "sandbox.env literal forwarded as FLY_RESIDENT_ENV_*"); - assert.doesNotMatch( - core, - /FLY_RESIDENT_ENV_COMPANY_API_TOKEN/, - "secretEnv value/name must not be written to the toml", - ); + const core = derivedTomlFor(config, "core", repoRoot); + assert.match(core, /SANDBOX_BACKEND = "sprites"/); + assert.doesNotMatch(core, /FLY_SANDBOX_APP_NAME|FLY_BASE_IMAGE|FLY_RESIDENT_ENV_/); - const admin = derivedTomlFor(cfg, "admin", repoRoot); - assert.doesNotMatch(admin, /FLY_RESIDENT_ENV_TZ/); - assert.doesNotMatch(admin, /FLY_SANDBOX_APP_NAME = "acme-sb"/); + const admin = derivedTomlFor(config, "admin", repoRoot); + assert.doesNotMatch(admin, /SANDBOX_BACKEND|FLY_SANDBOX_APP_NAME|FLY_BASE_IMAGE|FLY_RESIDENT_ENV_/); }); test("the fly target routes security screen proxy configuration only to core", () => { @@ -179,7 +157,7 @@ test("--only rejects a name that is neither a service nor a plugin (before any F ); }); -import { chmodSync, existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { flyPinSandbox, flyRollback, flySecretsPush } from "../src/backends/fly.ts"; function fakeFly(dir: string, script: string): { log: string; restore: () => void } { @@ -1084,261 +1062,10 @@ else console.log("ok");`, } }); -test("fly rollback joins a bare digest with @ and records the pin in the config file", () => { - const dir = mkdtempSync(join(tmpdir(), "qm-fly-rollback-")); - const appPrefix = "qmrbtest"; - const digest = `sha256:${"d".repeat(64)}`; - const config: QmConfig = { - contract: 1, - orgId: "acme", - publicUrl: "https://acme.example.com", - target: "fly", - appPrefix, - region: "sjc", - flyOrg: "personal", - services: ["core"], - plugins: [], - skills: [], - env: {}, - imageOverrides: {}, - sandbox: { app: `${appPrefix}-sb` }, - }; - const configPath = join(dir, "qm.config.jsonc"); - writeFileSync( - configPath, - `${JSON.stringify({ contract: 1, orgId: "acme", sandbox: { app: `${appPrefix}-sb` } }, null, 2)}\n`, - ); - const marker = `QM_OWNER_${createHash("sha256").update(`qm-v2:personal:acme:${appPrefix}`).digest("hex").slice(0, 16).toUpperCase()}`; - const fake = fakeFly( - dir, - ` -if (a.startsWith("apps list")) console.log(JSON.stringify([{ Name: "${appPrefix}-core" }])); -else if (a.startsWith("secrets list")) console.log("NAME DIGEST CREATED AT\\n${marker} abc123 1m ago"); -else if (a.startsWith("status")) console.log(JSON.stringify({ Machines: [{ id: "machine-core", config: { image: "registry.fly.io/${appPrefix}-core:cur" } }] })); -else if (a.startsWith("image show")) console.log(JSON.stringify([{ MachineID: "machine-core", Registry: "registry.fly.io", Repository: "${appPrefix}-core", Tag: "cur", Digest: "sha256:${"a".repeat(64)}" }])); -else console.log("");`, - ); - const resolved = `sha256:${"e".repeat(64)}`; - const docker = join(dir, "docker"); - writeFileSync(docker, `#!/usr/bin/env bash\necho "Digest: ${resolved}"\n`); - chmodSync(docker, 0o755); - const priorPath = process.env.PATH; - process.env.PATH = `${dir}:${priorPath ?? ""}`; - const generated = join(dir, ".generated", "fly", appPrefix); - const log = console.log; - console.log = (): void => {}; - try { - flyRollback(config, configPath, digest); - const toml = readFileSync(join(generated, "core.fly.toml"), "utf8"); - assert.ok( - toml.includes(`FLY_BASE_IMAGE = "registry.fly.io/${appPrefix}-sb@${digest}"`), - "digest joins the repository with @, not :", - ); - const pinned = JSON.parse(readFileSync(configPath, "utf8")) as { sandbox?: { image?: string } }; - assert.equal( - pinned.sandbox?.image, - `registry.fly.io/${appPrefix}-sb@${digest}`, - "the pin is durable in the config file", - ); - - flyRollback(config, configPath, "v12"); - const tagged = readFileSync(join(generated, "core.fly.toml"), "utf8"); - assert.ok( - tagged.includes(`FLY_BASE_IMAGE = "registry.fly.io/${appPrefix}-sb:v12@${resolved}"`), - "a tag is resolved to its immutable digest", - ); - const repinned = JSON.parse(readFileSync(configPath, "utf8")) as { sandbox?: { image?: string } }; - assert.equal( - repinned.sandbox?.image, - `registry.fly.io/${appPrefix}-sb:v12@${resolved}`, - "a tag pin would never compare stale; only a digest may land in the config", - ); - } finally { - console.log = log; - if (priorPath === undefined) delete process.env.PATH; - else process.env.PATH = priorPath; - fake.restore(); - rmSync(dir, { recursive: true, force: true }); - if (existsSync(generated)) rmSync(generated, { recursive: true, force: true }); - } -}); - -test("fly rollback with no sandbox config fails cleanly instead of deriving a garbage ref", () => { - const config: QmConfig = { - contract: 1, - orgId: "acme", - publicUrl: "https://acme.example.com", - target: "fly", - region: "sjc", - flyOrg: "personal", - services: ["core"], - plugins: [], - skills: [], - env: {}, - imageOverrides: {}, - }; - assert.throws( - () => flyRollback(config, "/nonexistent/config.json", `sha256:${"e".repeat(64)}`), - /cannot derive an image repository/, - ); -}); - -test("fly rollback validates an image pin before changing the committed config", () => { - const dir = mkdtempSync(join(tmpdir(), "qm-fly-rollback-invalid-")); - const configPath = join(dir, "qm.config.jsonc"); - const raw = JSON.stringify({ contract: 1, orgId: "acme", sandbox: { app: "acme-sb" } }, null, 2) + "\n"; - writeFileSync(configPath, raw); - const config: QmConfig = { - contract: 1, - orgId: "acme", - publicUrl: "https://acme.example.com", - target: "fly", - services: ["core"], - plugins: [], - skills: [], - env: {}, - imageOverrides: {}, - sandbox: { app: "acme-sb" }, - }; - try { - assert.throws( - () => flyRollback(config, configPath, "sha256:not-a-digest"), - /must resolve to an image tag or sha256 digest/, - ); - assert.equal(readFileSync(configPath, "utf8"), raw); - } finally { - rmSync(dir, { recursive: true, force: true }); - } -}); - -test("fly rollback refuses to change the committed config when the image cannot be resolved", () => { - const dir = mkdtempSync(join(tmpdir(), "qm-fly-rollback-missing-")); - const configPath = join(dir, "qm.config.jsonc"); - const raw = JSON.stringify({ contract: 1, orgId: "acme", sandbox: { app: "acme-sb" } }, null, 2) + "\n"; - writeFileSync(configPath, raw); - const docker = join(dir, "docker"); - writeFileSync(docker, "#!/usr/bin/env bash\necho missing >&2\nexit 1\n"); - chmodSync(docker, 0o755); - const priorPath = process.env.PATH; - process.env.PATH = `${dir}:${priorPath ?? ""}`; - const config: QmConfig = { - contract: 1, - orgId: "acme", - publicUrl: "https://acme.example.com", - target: "fly", - services: ["core"], - plugins: [], - skills: [], - env: {}, - imageOverrides: {}, - sandbox: { app: "acme-sb" }, - }; - try { - assert.throws(() => flyRollback(config, configPath, "v12"), /could not resolve an immutable digest/); - assert.equal(readFileSync(configPath, "utf8"), raw); - } finally { - if (priorPath === undefined) delete process.env.PATH; - else process.env.PATH = priorPath; - rmSync(dir, { recursive: true, force: true }); - } -}); - -test("fly sandbox pin re-derives the core [env] and unsets any stale FLY_BASE_IMAGE secret", () => { - const dir = mkdtempSync(join(tmpdir(), "qm-fly-pin-")); - const appPrefix = "qmpintest"; - const pin = `registry.fly.io/${appPrefix}-sb@sha256:${"c".repeat(64)}`; - const config: QmConfig = { - contract: 1, - orgId: "acme", - publicUrl: "https://acme.example.com", - target: "fly", - appPrefix, - region: "sjc", - flyOrg: "personal", - services: ["core"], - plugins: [], - skills: [], - env: {}, - imageOverrides: {}, - sandbox: { app: `${appPrefix}-sb`, image: pin }, - }; - const marker = `QM_OWNER_${createHash("sha256").update(`qm-v2:personal:acme:${appPrefix}`).digest("hex").slice(0, 16).toUpperCase()}`; - const fake = fakeFly( - dir, - ` -if (a.startsWith("apps list")) console.log(JSON.stringify([{ Name: "${appPrefix}-core" }])); -else if (a.startsWith("status")) console.log(JSON.stringify({ Machines: [{ id: "machine-core", config: { image: "registry.fly.io/${appPrefix}-core:cur" } }] })); -else if (a.startsWith("image show")) console.log(JSON.stringify([{ MachineID: "machine-core", Registry: "registry.fly.io", Repository: "${appPrefix}-core", Tag: "cur", Digest: "sha256:${"a".repeat(64)}" }])); -else if (a.startsWith("secrets list")) console.log("NAME DIGEST CREATED AT\\n${marker} abc123 1m ago\\nFLY_BASE_IMAGE abc123 1m ago"); -else console.log("");`, - ); - const generated = join(dir, ".generated", "fly", appPrefix); - const log = console.log; - console.log = (): void => {}; - try { - flyPinSandbox(config, pin, dir); - const calls = readFileSync(fake.log, "utf8"); - assert.ok( - calls.includes(`secrets unset --stage -a ${appPrefix}-core FLY_BASE_IMAGE`), - "the stale shadowing secret is removed", - ); - assert.ok(!/secrets set/.test(calls), "the pin is never staged as a Fly secret"); - const deploy = calls.split("\n").find((line) => line.startsWith("deploy")); - assert.ok( - deploy?.includes(`--image registry.fly.io/${appPrefix}-core@sha256:${"a".repeat(64)}`), - `rolls the current immutable image: ${deploy}`, - ); - const toml = readFileSync(join(generated, "core.fly.toml"), "utf8"); - assert.ok(toml.includes(`FLY_BASE_IMAGE = "${pin}"`), "the pin lands in the derived [env], owned by the config"); - assert.ok(deploy?.includes(join(generated, "core.fly.toml")), "deploy uses the derived config"); - } finally { - console.log = log; - fake.restore(); - rmSync(dir, { recursive: true, force: true }); - if (existsSync(generated)) rmSync(generated, { recursive: true, force: true }); - } -}); - -test("fly sandbox pin refuses to mutate an unmarked app with the configured name", () => { - const dir = mkdtempSync(join(tmpdir(), "qm-fly-pin-unowned-")); - const appPrefix = "qmpinunowned"; - const pin = `registry.fly.io/${appPrefix}-sb@sha256:${"d".repeat(64)}`; - const config: QmConfig = { - contract: 1, - orgId: "acme", - publicUrl: "https://acme.example.com", - target: "fly", - appPrefix, - region: "sjc", - flyOrg: "personal", - services: ["core"], - plugins: [], - skills: [], - env: {}, - imageOverrides: {}, - sandbox: { app: `${appPrefix}-sb`, image: pin }, - }; - const fake = fakeFly( - dir, - ` -if (a.startsWith("apps list")) console.log(JSON.stringify([{ Name: "${appPrefix}-core" }])); -else if (a.startsWith("secrets list")) console.log("NAME DIGEST CREATED AT"); -else console.log("");`, - ); - const log = console.log; - console.log = (): void => {}; - try { - assert.throws( - () => flyPinSandbox(config, pin), - /not marked as owned by deployment qm-v2:personal:acme:qmpinunowned/, - ); - const calls = readFileSync(fake.log, "utf8"); - assert.doesNotMatch(calls, /secrets unset|deploy/, "an unowned app is never mutated"); - } finally { - console.log = log; - fake.restore(); - rmSync(dir, { recursive: true, force: true }); - } +test("fly sandbox pin and rollback reject unsupported Sprites images", () => { + const config = {} as QmConfig; + assert.throws(() => flyPinSandbox(config, "registry.fly.io/acme-sb:latest"), /stock runtime.*image pins/); + assert.throws(() => flyRollback(config, "/nonexistent/config.json"), /rollback is unavailable.*image pins/); }); test("fly secrets push stages a secretEnv alias under its declared env name on its service's app", async () => { diff --git a/cli/test/init.test.ts b/cli/test/init.test.ts index 1c334e40a..dbe161905 100644 --- a/cli/test/init.test.ts +++ b/cli/test/init.test.ts @@ -141,6 +141,7 @@ test("init --target fly scaffolds the full hosted topology and both Slack apps", assert.deepEqual(config.services, ["core", "slack", "web-ui", "admin", "portal", "auth"]); assert.equal(config.publicUrl, "https://acme-portal.fly.dev"); assert.equal(config.flyOrg, "personal"); + assert.equal(config.sandbox, undefined); assert.ok(config.region, "region is scaffolded"); assert.equal(config.appPrefix, "acme"); assert.deepEqual( @@ -175,6 +176,10 @@ test("init --target fly scaffolds the full hosted topology and both Slack apps", assert.ok(env.split("\n").includes(line), `.env.example should offer ${line}`); } assert.ok(existsSync(join(dir, "slack-app-manifest.yml")), "Slack manifest is scaffolded on fly too"); + assert.equal(existsSync(join(dir, "sandbox", "tools")), false); + const greet = readFileSync(join(dir, "sandbox", "skills", "greet", "SKILL.md"), "utf8"); + assert.doesNotMatch(greet, /example-tool/); + assert.deepEqual(validateSandboxLayer(join(dir, "sandbox"), config).errors, []); assert.equal(existsSync(join(dir, "slack-sso-manifest.yml")), false); for (const line of ["# OIDC_CLIENT_ID=", "# OIDC_CLIENT_SECRET=", "# PORTAL_EXPECTED_TEAM_ID="]) { assert.ok(env.split("\n").includes(line), `external-IdP secret ${line} stays documented but unrequired`); diff --git a/cli/test/package.test.ts b/cli/test/package.test.ts index e4b9c0e06..8f4144692 100644 --- a/cli/test/package.test.ts +++ b/cli/test/package.test.ts @@ -46,6 +46,7 @@ test( ) as Array<{ filename: string; files: Array<{ path: string }> }>; const tarball = join(dir, packed[0]!.filename); const deployment = join(dir, "deployment"); + const dockerDeployment = join(dir, "docker-deployment"); const awsDeployment = join(dir, "aws-deployment"); const tarballBytes = readFileSync(tarball); const packageManifest = JSON.parse(readFileSync(join(cliDir, "package.json"), "utf8")) as Record; @@ -84,6 +85,7 @@ test( const registryUrl = `http://127.0.0.1:${(registry.address() as AddressInfo).port}/`; const consumers = [ { dir: deployment, org: "acme", target: "fly" }, + { dir: dockerDeployment, org: "acme-docker", target: "docker" }, { dir: awsDeployment, org: "acme-aws", target: "aws" }, ] as const; for (const consumer of consumers) { @@ -140,15 +142,16 @@ test( await new Promise((resolve, reject) => registry!.close((error) => (error ? reject(error) : resolve()))); registry = undefined; const bin = join(deployment, "node_modules", ".bin", "qm"); + const dockerBin = join(dockerDeployment, "node_modules", ".bin", "qm"); const awsBin = join(awsDeployment, "node_modules", ".bin", "qm"); rmSync(tarball); - const deploymentConfig = join(deployment, "qm.config.jsonc"); + const dockerConfig = join(dockerDeployment, "qm.config.jsonc"); writeFileSync( - deploymentConfig, - readFileSync(deploymentConfig, "utf8").replace( - '"sandbox": { "app": "acme-sandboxes" }', - `"sandbox": { "app": "acme-sandboxes", "image": "registry.fly.io/acme-sandboxes@sha256:${"a".repeat(64)}" }`, + dockerConfig, + readFileSync(dockerConfig, "utf8").replace( + '"sandbox": { "app": "acme-docker-sandboxes" }', + `"sandbox": { "app": "acme-docker-sandboxes", "image": "registry.fly.io/acme-docker-sandboxes@sha256:${"a".repeat(64)}" }`, ), ); @@ -175,8 +178,8 @@ test( assert.match(execFileSync(bin, ["version"], { encoding: "utf8" }), /^\d+\.\d+\.\d+/); assert.match(execFileSync(bin, ["check"], { cwd: deployment, encoding: "utf8", env }), /check passed/); - const dockerPlan = execFileSync(bin, ["plan", "--target", "docker"], { - cwd: deployment, + const dockerPlan = execFileSync(dockerBin, ["plan"], { + cwd: dockerDeployment, encoding: "utf8", env, }); @@ -195,8 +198,16 @@ test( const generatedCore = join(deployment, ".generated", "fly", "acme", "core.fly.toml"); assert.ok(existsSync(generatedCore)); assert.doesNotMatch(readFileSync(generatedCore, "utf8"), /^\s*PI_(?:MODEL|DETECT_MODEL)\s*=/m); + assert.throws( + () => execFileSync(bin, ["sandbox", "publish", "--dry-run"], { cwd: deployment, encoding: "utf8", env }), + /stock runtime/, + ); assert.match( - execFileSync(bin, ["sandbox", "publish", "--dry-run"], { cwd: deployment, encoding: "utf8", env }), + execFileSync(dockerBin, ["sandbox", "publish", "--dry-run"], { + cwd: dockerDeployment, + encoding: "utf8", + env, + }), /qm-sandbox-base@sha256:a{64}/, ); const outputs = JSON.parse( diff --git a/cli/test/sandbox-build.test.ts b/cli/test/sandbox-build.test.ts index 75a052dfc..ace0ecae5 100644 --- a/cli/test/sandbox-build.test.ts +++ b/cli/test/sandbox-build.test.ts @@ -150,3 +150,19 @@ test("a broken layer (tool with no executable and no Dockerfile) fails before bu rmSync(sb, { recursive: true, force: true }); } }); + +test("Fly Sprites reject sandbox image builds before inspecting the layer", () => { + const sb = sandboxDir(() => undefined); + const config: QmConfig = { + ...CONFIG, + target: "fly", + publicUrl: "https://acme.example.com", + region: "sjc", + flyOrg: "personal", + }; + try { + assert.throws(() => dryRun({ sandboxDir: sb, config }), /Fly Sprites use the stock runtime/); + } finally { + rmSync(sb, { recursive: true, force: true }); + } +}); diff --git a/cli/test/sandbox-publish.test.ts b/cli/test/sandbox-publish.test.ts index 1dcc5d113..b1a4ae6e3 100644 --- a/cli/test/sandbox-publish.test.ts +++ b/cli/test/sandbox-publish.test.ts @@ -708,7 +708,7 @@ test("AWS sandbox publish preconditions run before the build: frozen override an publicUrl: "https://agent.acme.example", target: "docker", services: ["core"], - sandbox: { backend: "sprites", app: "acme-sandboxes", image: pin }, + sandbox: { app: "acme-sandboxes", image: pin }, }); writeFileSync(configPath, configBody); const toolDir = join(dir, "sandbox", "tools", "t"); @@ -733,11 +733,11 @@ test("AWS sandbox publish preconditions run before the build: frozen override an config: frozen.config, configPath, }), - /freezes the sandbox pin/, + /custom images are unsupported/, ); const pinless: DeployContext = { ...frozen, - config: { ...frozen.config, sandbox: { backend: "sprites", app: "acme-sandboxes" } }, + config: { ...frozen.config, sandbox: { app: "acme-sandboxes" } }, }; await assert.rejects( () => @@ -746,7 +746,7 @@ test("AWS sandbox publish preconditions run before the build: frozen override an config: pinless.config, configPath, }), - /no AWS deployment manifest exists yet/, + /custom images are unsupported/, ); assert.equal(readFileSync(dockerLog, "utf8"), "", "preconditions fail before any docker side effect"); assert.equal(readFileSync(configPath, "utf8"), configBody); @@ -757,7 +757,7 @@ test("AWS sandbox publish preconditions run before the build: frozen override an } }); -test("sandbox publish on AWS with sandbox.backend sprites proceeds to the layer build (operator-hosted sandbox image)", async () => { +test("the AWS provider rejects the removed Sprites image mode before inspecting the layer", async () => { const dir = mkdtempSync(join(tmpdir(), "qm-publish-aws-fly-")); const aws = fakeAwsBin(dir, { id: "m1", @@ -772,7 +772,7 @@ test("sandbox publish on AWS with sandbox.backend sprites proceeds to the layer publicUrl: "https://agent.acme.example", target: "docker", services: ["core"], - sandbox: { backend: "sprites", app: "acme-sandboxes" }, + sandbox: { app: "acme-sandboxes" }, }); writeFileSync(configPath, configBody); const { config } = loadConfigAt(configPath); @@ -785,7 +785,7 @@ test("sandbox publish on AWS with sandbox.backend sprites proceeds to the layer }; await assert.rejects( () => hostingProvider("aws").publishSandbox(ctx, { sandboxDir: ctx.sandboxDir, config: ctx.config, configPath }), - /nothing to build/, + /custom images are unsupported/, ); assert.equal(readFileSync(configPath, "utf8"), configBody); } finally { diff --git a/cli/test/secrets.test.ts b/cli/test/secrets.test.ts index e44035be3..6fe14149e 100644 --- a/cli/test/secrets.test.ts +++ b/cli/test/secrets.test.ts @@ -159,22 +159,17 @@ test("model credentials are optional at deploy time because Admin onboarding can assert.equal(secretByName(docker, "ANTHROPIC_API_KEY").required, false); }); -test("the Fly sandbox token avoids flyctl's FLY_API_TOKEN authentication variable", () => { - const fly = makeConfig({ target: "fly" }); - const sandbox = secretByName(fly, "FLY_SANDBOX_API_TOKEN"); - assert.ok(sandbox.required); - assert.deepEqual(runtimeSecretNames("core", sandbox), ["FLY_API_TOKEN"]); - assert.ok(!computedSecrets(fly).some((secret) => secret.name === "FLY_API_TOKEN")); +test("the Fly target does not request a token for an unsupported custom sandbox app", () => { + assert.ok(!computedSecrets(makeConfig({ target: "fly" })).some((secret) => secret.name === "FLY_SANDBOX_API_TOKEN")); }); -test("the Fly tokens belong to a Fly target, and the publisher token only to a Fly deploy provider", () => { - assert.ok(secretByName(makeConfig({ target: "fly" }), "FLY_SANDBOX_API_TOKEN").required); +test("the Fly deploy publisher token only belongs to an enabled Fly deploy provider", () => { assert.ok(!computedSecrets(makeConfig({ target: "fly" })).some((secret) => secret.name === "FLY_DEPLOY_API_TOKEN")); assert.ok( secretByName(makeConfig({ target: "fly", env: { core: { DEPLOY_PROVIDER: "fly" } } }), "FLY_DEPLOY_API_TOKEN") .required, ); - for (const config of [makeConfig(), makeConfig({ sandbox: { app: "acme-sb" } }), makeConfig({ target: "aws" })]) { + for (const config of [makeConfig(), makeConfig({ target: "aws" })]) { assert.ok(!computedSecrets(config).some((secret) => secret.name === "FLY_SANDBOX_API_TOKEN")); assert.ok(!computedSecrets(config).some((secret) => secret.name === "FLY_DEPLOY_API_TOKEN")); } diff --git a/deploy/core/fly.toml b/deploy/core/fly.toml index d8beef05f..78fc1b91f 100644 --- a/deploy/core/fly.toml +++ b/deploy/core/fly.toml @@ -15,12 +15,9 @@ primary_region = "sjc" S3_REGION = "auto" DATA_DIR = "/data" PUBLIC_WEB_URL = "https://agent.example.com" - FLY_SANDBOX_APP_NAME = "acme-sandboxes" - FLY_BASE_IMAGE = "registry.fly.io/acme-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a" WEB_UI_PUBLIC_URL = "https://agent.example.com" REQUIRE_SIGNED_PORTAL_IDENTITY = "1" FLY_ORG = "personal" - FLY_DEPLOY_BASE_IMAGE = "registry.fly.io/acme-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a" FLY_DEPLOY_APP_PREFIX = "qm-d" SANDBOX_BACKEND = "sprites" [deploy] diff --git a/deploy/egress-proxy/Dockerfile b/deploy/egress-proxy/Dockerfile index 6c9b077a1..038e143a5 100644 --- a/deploy/egress-proxy/Dockerfile +++ b/deploy/egress-proxy/Dockerfile @@ -14,6 +14,9 @@ COPY src ./src COPY tsconfig.json ./ COPY deploy/egress-proxy/envoy.yaml deploy/egress-proxy/start.sh ./ RUN chmod +x start.sh +# A rejected config is a fleet-wide outage: start.sh kills the container when Envoy dies, so an +# invalid envoy.yaml has to fail the build, not the deploy. +RUN envoy --mode validate -c /app/envoy.yaml ENV NODE_ENV=production EXPOSE 48080 diff --git a/deploy/egress-proxy/envoy.yaml b/deploy/egress-proxy/envoy.yaml index a2074c6a0..1e2ca2b37 100644 --- a/deploy/egress-proxy/envoy.yaml +++ b/deploy/egress-proxy/envoy.yaml @@ -6,6 +6,16 @@ # # Cloud metadata authorities (IMDS 169.254.169.254, GCP metadata, AWS IPv6 IMDS) are ALSO denied # statically here at the Envoy layer — belt and braces. + +# A UNIX SOCKET, never a TCP port — not even loopback. This process is a forward proxy: a +# sandbox that asks it to `CONNECT 127.0.0.1:` makes it dial its own namespace, so a +# loopback admin listener is reachable by every sandbox and /quitquitquit would drop fleet +# egress on demand. The decision service denies loopback destinations too; this is the other half. +# Read it with `curl --unix-socket /tmp/envoy-admin.sock http://localhost/stats`. mode 384 = 0600. +admin: + address: + pipe: { path: /tmp/envoy-admin.sock, mode: 384 } + static_resources: listeners: - name: egress_proxy @@ -17,6 +27,17 @@ static_resources: typed_config: "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager stat_prefix: egress_proxy + # One line per connection, on stdout (`fly logs`). %RESPONSE_FLAGS% and + # %RESPONSE_CODE_DETAILS% are the point: they name WHY a request failed + # (lua_response, upstream_reset_before_response_started{…}), which the + # 503's body carries but a failed CONNECT throws away. + access_log: + - name: envoy.access_loggers.stdout + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog + log_format: + text_format_source: + inline_string: "[envoy] %REQ(:METHOD)% %REQ(:AUTHORITY):256% %RESPONSE_CODE% flags=%RESPONSE_FLAGS% detail=%RESPONSE_CODE_DETAILS% upstream=%UPSTREAM_HOST% dur=%DURATION%\n" http_protocol_options: allow_absolute_url: true upgrade_configs: diff --git a/deploy/egress-proxy/fly.toml b/deploy/egress-proxy/fly.toml index e039ca0af..507e936a4 100644 --- a/deploy/egress-proxy/fly.toml +++ b/deploy/egress-proxy/fly.toml @@ -25,9 +25,11 @@ primary_region = "sjc" timeout = "2s" grace_period = "10s" +# Envoy + the Node decision service sit at ~130mb resident before serving anything, +# which leaves almost no headroom on 256mb under real fleet traffic. [[vm]] size = "shared-cpu-1x" - memory = "256mb" + memory = "512mb" kill_signal = "SIGTERM" kill_timeout = "10s" diff --git a/deploy/stacks/acme/qm.config.jsonc b/deploy/stacks/acme/qm.config.jsonc index 1153bb351..97429f42f 100644 --- a/deploy/stacks/acme/qm.config.jsonc +++ b/deploy/stacks/acme/qm.config.jsonc @@ -7,10 +7,6 @@ "region": "sjc", "flyOrg": "personal", "deployAppPrefix": "qm-d", - "sandbox": { - "app": "acme-sandboxes", - "image": "registry.fly.io/acme-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a", - }, "services": ["core", "slack", "admin", "web-ui", "portal"], "env": { "admin": { diff --git a/deploy/stacks/broker/qm.config.jsonc b/deploy/stacks/broker/qm.config.jsonc index 9e8958c08..fc5ca33a1 100644 --- a/deploy/stacks/broker/qm.config.jsonc +++ b/deploy/stacks/broker/qm.config.jsonc @@ -11,10 +11,6 @@ "region": "sjc", "flyOrg": "personal", "deployAppPrefix": "qm-d", - "sandbox": { - "app": "acme-sandboxes", - "image": "registry.fly.io/acme-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a", - }, "services": ["core", "slack", "admin", "web-ui", "portal", "auth"], "env": { "admin": { diff --git a/deploy/stacks/demo.json b/deploy/stacks/demo.json index 64003f711..7de38e46a 100644 --- a/deploy/stacks/demo.json +++ b/deploy/stacks/demo.json @@ -7,10 +7,6 @@ "region": "sjc", "flyOrg": "personal", "imageFrom": "qm", - "sandbox": { - "app": "demo-agent-sandboxes", - "image": "registry.fly.io/demo-agent-sandboxes@sha256:1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a" - }, "services": ["core", "admin", "web-ui", "portal"], "vms": { "core": { "size": "shared-cpu-2x", "memory": "4gb" } @@ -19,7 +15,6 @@ "core": { "WEB_UI_PUBLIC_URL": "https://demo-agent.example.com/web-ui", "PUBLIC_WEB_URL": "https://demo-agent.example.com", - "FLY_SANDBOX_APP_NAME": "demo-agent-sandboxes", "SNAPSHOT_STORE": "s3", "TRANSFER_STORE": "s3", "S3_BUCKET": "demo-agent-data", diff --git a/docs/deploy-directory.md b/docs/deploy-directory.md index a020f7e02..1cd9d26d5 100644 --- a/docs/deploy-directory.md +++ b/docs/deploy-directory.md @@ -4,7 +4,7 @@ Contract v1 makes a QM deployment a committed, portable directory. The `qm` CLI ## Layout -`package.json` pins the `@yc-software/qm` deployment engine at the exact version that scaffolded the directory, so the directory records which CLI interprets it rather than drifting with whatever version an operator has installed; `contract: 1` remains only the compatibility floor. `package-lock.json` records the installed artifact. `qm.config.jsonc` is the deployment config. `deployment.md` and `.codex/skills/deploy-qm/` are materialized package assets an operator can hand to an agent. `sandbox/` adds tools and skills to agent computers; `plugins/` adds services; `.env.example` documents the computed secret names; `.env` supplies local values and is never committed. `qm init` writes `slack-app-manifest.yml` for the optional Socket Mode bot. It also writes `slack-sso-manifest.yml` only when the portal is configured to use Slack OpenID. `qm slack render` refreshes the applicable manifests after `publicUrl` changes, and `qm outputs` returns their creation links and the web coordinates. `qm init --target aws` also vendors the reference `infra/` Terraform module and its derived `terraform.tfvars`; the copy belongs to the deployment after generation. Init never overwrites an existing deployment config. +`package.json` pins the `@yc-software/qm` deployment engine at the exact version that scaffolded the directory, so the directory records which CLI interprets it rather than drifting with whatever version an operator has installed; `contract: 1` remains only the compatibility floor. `package-lock.json` records the installed artifact. `qm.config.jsonc` is the deployment config. `deployment.md` and `.codex/skills/deploy-qm/` are materialized package assets an operator can hand to an agent. `sandbox/` adds skills on every target and image-installed tools where the target supports them; `plugins/` adds services; `.env.example` documents the computed secret names; `.env` supplies local values and is never committed. `qm init` writes `slack-app-manifest.yml` for the optional Socket Mode bot. It also writes `slack-sso-manifest.yml` only when the portal is configured to use Slack OpenID. `qm slack render` refreshes the applicable manifests after `publicUrl` changes, and `qm outputs` returns their creation links and the web coordinates. `qm init --target aws` also vendors the reference `infra/` Terraform module and its derived `terraform.tfvars`; the copy belongs to the deployment after generation. Init never overwrites an existing deployment config. The sandbox layout is: @@ -17,13 +17,13 @@ sandbox/ skills// ``` -The Dockerfile is optional when every declared binary is present in its tool directory. Skill assets delivered through the deployment-layer API are text in v1; binaries belong in the sandbox image. +The Dockerfile is optional when every declared binary is present in its tool directory. Skill assets delivered through the deployment-layer API are text in v1; binaries belong in a supported sandbox image. Fly rejects the Dockerfile and `tools/` while accepting `skills/`. ## Configuration -The root object requires `contract: 1`, `orgId`, `publicUrl`, `target`, and `services` including `core`. Docker and Fly also require `sandbox.app`. On AWS the sandbox substrate is an explicit choice: omitting the `sandbox` block runs named Lambda MicroVM images; declaring one requires `sandbox.backend` — `"sprites"` boots the operator-published layer image in `sandbox.app`, `"aws"` states the MicroVM default in the file. Unknown contract majors fail closed. `target` is `docker`, `fly`, or `aws`. +The root object requires `contract: 1`, `orgId`, `publicUrl`, `target`, and `services` including `core`. Docker requires `sandbox.app` for its OCI image repository. Fly uses the stock Sprites runtime and may omit `sandbox` or declare only `sandbox.backend: "sprites"`. AWS uses Lambda MicroVM images and may omit `sandbox` or declare only `sandbox.backend: "aws"`. Unknown contract majors fail closed. `target` is `docker`, `fly`, or `aws`. -Common optional fields select the model, plugins, extra skill directories, per-service non-secret environment values, image overrides, sandbox settings, and an external security screen. `sandbox.backend` selects the aws-target sandbox substrate (see above); `sandbox.image` is the immutable rootfs pin used at boot; `sandbox.baseImage` records the digest-pinned build input; `sandbox.env` is non-secret runtime environment; `sandbox.secretEnv` lists org-wide secret names whose values are forwarded to every sandbox. `securityScreen` contains `backend: "proxy"`, a lowercase provider label, an HTTPS endpoint, and a `shadow` or `enforce` rollout. Its presence requires `secretEnv.core.SECURITY_SCREEN_PROXY_TOKEN`; absence keeps Auto on the built-in model classifier. +Common optional fields select the model, plugins, extra skill directories, per-service non-secret environment values, image overrides, sandbox settings, and an external security screen. On Docker, `sandbox.image` is the immutable rootfs pin used at boot, `sandbox.baseImage` records the digest-pinned build input, `sandbox.env` is non-secret runtime environment, and `sandbox.secretEnv` lists secrets forwarded to the sandbox. Fly rejects all five fields because the installed Sprites SDK has no image or resident-environment materialization path. AWS rejects them because its guest image is managed by `infra build-image`. `securityScreen` contains `backend: "proxy"`, a lowercase provider label, an HTTPS endpoint, and a `shadow` or `enforce` rollout. Its presence requires `secretEnv.core.SECURITY_SCREEN_PROXY_TOKEN`; absence keeps Auto on the built-in model classifier. Fly requires `region` and `flyOrg`. AWS requires a 12-digit account, region, deployment label, ECS cluster, deploy-role ARN, Secrets Manager prefix, DNS-valid Cloud Map namespace, and an entry for every enabled first-party service and discovered plugin containing a unique valid ECR repository, a unique valid ECS service, and a valid Fargate CPU/memory combination. The cluster is constrained so every IAM, RDS, ALB, and related name derived by the reference module is valid. `imageLabel` identifies the complete deployment manifest used by rollback and live drift checks; the matching OCI/ECR tag is a convenience pointer. Workloads may also set `arm64`/`amd64` architecture, non-secret build arguments, or role ARNs. External prebuilt images must declare their architecture; source-built and built-in workloads use their platform default. Cloud Map names are the private workload addresses. The reference AWS module exposes CloudFront over HTTPS and restricts its HTTP ALB origin to CloudFront's managed origin prefix. With portal enabled, it is the ALB's sole target; access to private core, web, and admin surfaces requires signed portal identity. Without portal, only core is an ALB target. A real harness requires an HTTPS `publicUrl`. @@ -70,7 +70,7 @@ A chunk whose score is at or above its threshold resolves to Strict, and any Str ## Secrets -First-party services publish a typed `SecretSpec` schema. The CLI combines the enabled services and feature predicates with plugin `secrets` and `sandbox.secretEnv` to form the computed secret set. That same schema determines which task receives each secret. Core validates its own required runtime secrets at production boot. +First-party services publish a typed `SecretSpec` schema. The CLI combines the enabled services and feature predicates with plugin `secrets` and Docker's optional `sandbox.secretEnv` to form the computed secret set. That same schema determines which task receives each secret. Core validates its own required runtime secrets at production boot. `init` renders the set as `.env.example`; that file has names and descriptions, never values, and is not an input to deployment. Operators place values in gitignored `.env`. Docker reads the file locally. `qm secrets push` uploads supplied operator-managed values to Fly secrets or AWS Secrets Manager without printing them. Terraform owns `DATABASE_URL` on AWS because it owns RDS. `doctor` treats missing and placeholder required values as failures and reports absent optional plugin secrets without blocking deployment. @@ -98,7 +98,7 @@ Deployment-specific safety belongs here too. For example, an ambiently authentic When `sandbox/` exists, every `up` sends its descriptors and complete text skill trees to source-authenticated `PUT /v1/deployment-layer`. Without `sandbox/`, `up` skips layer sync and leaves the deployed layer unchanged. Core validates submitted bundles again, stores them in Postgres table `deployment_layer`, versions them by a canonical SHA-256 content hash, records an audit event, hydrates them before serving, and returns the restorable bundle with its metadata and resolved runtime state from source-authenticated `GET /v1/deployment-layer`. Removed layer-owned skills are archived. Filesystem `DEPLOYMENT_LAYER` remains a bootstrap input for local and recovery use. -The sandbox handoff is a substrate image pin plus a layer content hash. Docker and Fly use `sandbox publish` to push an OCI image, resolve its immutable digest, and record it in the config. AWS with `sandbox.backend: "aws"` (or no sandbox block) uses `infra build-image` to package the guest agent as a Lambda MicroVM image and records its immutable image version and execution role; with `sandbox.backend: "sprites"`, `sandbox publish` pushes the layer image and records its digest pin in the durable deployment manifest, which `up`, `check --live`, and `rollback` resolve. Service task definitions and sandbox root filesystems use immutable pins, not mutable tags. +Docker uses `sandbox publish` to push an OCI sandbox image, resolve its immutable digest, and record it in the config. AWS uses `infra build-image` to package the guest agent as a Lambda MicroVM image and records its immutable image version and execution role. Fly uses the stock Sprites runtime; `sandbox build`, `sandbox publish`, custom Dockerfiles, tool binaries, image pins, and resident environment are rejected. Fly deployment layers may contain text skills only. Service task definitions and supported sandbox root filesystems use immutable pins, not mutable tags. Postgres stores create their tables lazily with idempotent DDL through the shared pool. The Terraform module creates RDS and its `DATABASE_URL` secret. On AWS, `up` takes a manual RDS snapshot before its first mutation — refusing an unavailable database or one whose automated-backup retention is below `aws.dbRetentionMinDays` (default 1) — named after the deployment manifest it precedes and recorded in that manifest; older pre-deploy snapshots are pruned to a bounded count, and `aws.predeployDbSnapshot: false` opts a deployment out. Restore remains operator-run: `rollback` prints the snapshot to restore alongside the code it rolls back. @@ -107,8 +107,8 @@ Postgres stores create their tables lazily with idempotent DDL through the share | Requirement | Docker | Fly | AWS | | ------------------------------------------------------------------------------------------ | ------------------------------: | ------------------------------: | ----------------------------------------------: | | Node 24 and `qm` CLI | yes | yes | yes | -| Docker daemon | yes | build path | image transfer/build path | -| Agent-computer image and credentials | Fly app for real execution | Fly app and scoped token | Lambda MicroVM image/version and execution role | +| Docker daemon | yes | no | image transfer/build path | +| Agent-computer runtime | pinned OCI image | stock Fly Sprites SDK | Lambda MicroVM image/version and execution role | | Slack bot app created from generated manifest, bot token, app token | when Slack enabled | when Slack enabled | when Slack enabled | | Admin email, verified sender, and a Resend key or SMTP credentials | with the built-in `auth` broker | with the built-in `auth` broker | with the built-in `auth` broker | | Slack SSO app, client id/secret, team gate, and exact `/auth/callback` redirect | only with Slack OIDC | only with Slack OIDC | only with Slack OIDC | @@ -119,7 +119,7 @@ Postgres stores create their tables lazily with idempotent DDL through the share ## Commands, conformance, and versioning -The normal gate order is `check`, `doctor`, substrate image build, `plan`, `up --yes`, then `check --live`. The substrate step is `sandbox publish` on Docker/Fly and `infra build-image` on AWS. First-party services come from the package's matching image manifest; `--build-from` is an explicit contributor escape hatch for unreleased source. `check` is static and has JSON output keyed to clause ids. `doctor` makes read-only external checks. `plan` renders without mutation. On AWS, rollback restores the prior recorded deployment manifest as one unit under the deployment lease; `--to` selects another complete manifest by manifest id or recorded release label. Because rollback restores code and configuration but never data, it prints the pre-deploy database snapshot recorded on the deployment it rolls back. On Fly it restores a sandbox pin. Docker does not claim rollback. +The normal gate order is `check`, `doctor`, any required substrate image build, `plan`, `up --yes`, then `check --live`. The substrate step is `sandbox publish` on Docker, none on Fly, and `infra build-image` on AWS. First-party services come from the package's matching image manifest; `--build-from` is an explicit contributor escape hatch for unreleased source. `check` is static and has JSON output keyed to clause ids. `doctor` makes read-only external checks. `plan` renders without mutation. On AWS, rollback restores the prior recorded deployment manifest as one unit under the deployment lease; `--to` selects another complete manifest by manifest id or recorded release label. Because rollback restores code and configuration but never data, it prints the pre-deploy database snapshot recorded on the deployment it rolls back. Fly and Docker do not claim sandbox rollback. AWS `up` is mutually excluded by a DynamoDB lease, snapshots the RDS instance before its first mutation, registers digest-pinned task definitions, enables the ECS circuit breaker, updates services, and waits stable. AWS `check --live` compares environment, secret routing, task definitions, sandbox pins, and the configured release label in both directions. Fly `check --live` verifies every configured workload has a live image-bearing machine and the public health endpoint responds. `qm conformance` remains the later cross-check between the static contract and core's resolved deployment-layer descriptors. diff --git a/fly/README.md b/fly/README.md index 10a4b6453..4b3b34c39 100644 --- a/fly/README.md +++ b/fly/README.md @@ -1,216 +1,15 @@ -# Fly sandbox +# Fly sandbox image assets -Per-scope **Firecracker microVMs** as the agent sandbox. -Stronger isolation than a container (separate kernel) and the machine **persists** -between turns, so installed packages / venv / build state stay warm — a private -"laptop" per scope. The core's `FlySandbox` (`src/sandbox/fly-sandbox.ts`) drives -machines over the [Fly Machines API](https://fly.io/docs/machines/api/). The VM disk -is the writable source of truth for this backend: the **read-only** mount layers -(org/team/granted scopes) are materialized into it from their owners' stores each turn, -while the **writable** layer lives on the persistent disk and is never re-seeded from -the store (cold-start recovery is `backupComputer`'s object-store backup). `snapshotWritable` -remains intentionally a no-op because the legacy per-turn workspace snapshot is the wrong -abstraction for a resident computer. +The Fly deployment target uses the stock runtime provided by the installed +`@fly/sprites` SDK. That SDK version can create and delete Sprites but has no +supported path for supplying an OCI image or persistent environment. -Because the snapshot is a no-op, the `write` primitive **dual-writes** each explicit write -to the durable workspace store (gated by the orchestrator's `persistWritesToStore`). A -cross-scope grant resolves a shared file from the **owner's** store, so without this an -agent-written file would be invisible to a grantee — "share this redline with Angela" -would silently fail. Transient spools (inbox/outbox/skills) and read-only mounts stay -VM-only. +Consequently these image assets are not part of the deployment-directory +contract. `qm` rejects Fly sandbox image fields, resident environment, +Dockerfiles, and tool binaries instead of accepting configuration it cannot +materialize. Deployments may still deliver text skills through the durable +deployment layer. -The base image is a Debian (glibc) + Node image (see `fly/Dockerfile`), with AWS CLI v2 -baked in; glibc means vendor install scripts and prebuilt binaries (AWS CLI v2, gcloud, -kubectl, gh) work as they do on a typical laptop, without musl compatibility shims. The -agent installs whatever else it needs on the resident disk, the way a colleague would. - -## When to use which sandbox - -- **docker** (default) — container, **open egress** (default bridge). Local dev + most prod. -- **local** — host `child_process`, fast, **not isolated**. Dev/tests only. -- **fly** — persistent microVM, warm state. **Open egress.** - -## Agent Computer profile - -Fly declares the first resident-auth Agent Computer profile: - -- `isolation=microvm` -- `lifetime=per_scope` -- `writablePersistence=resident_disk` -- `homePersistence=resident_disk` -- `egress=provider_governed` -- `auth=resident_machine` - -Docker/local remain per-turn sandboxes that snapshot writable files back to the -workspace store and do not support resident machine auth. - -## Durable `$HOME` volume & auto-upgrade - -Each scope's `$HOME` (`/root`) is a **per-scope Fly volume** (`home_`, sized by -`FLY_VOLUME_GB`, default 3 GiB), created on first provision and mounted into the machine. -This separates two lifecycles that used to be fused in one rootfs: - -- **Durable agent state** (resident creds and anything written under `$HOME`) lives on the - volume and survives a rootfs swap — like a laptop's home dir surviving an OS upgrade. -- **The rootfs** (OS + baked tools like `browse` and any deployment-layer CLIs, plus the baked Python venv at - `/opt/agent-venv`) can be replaced freely. (`pip install`s land in that venv, so they reset - on an image recreate — an accepted, rare cost.) - -So a new base image rolls out **without a manual `fly machine destroy`**. Core receives the -target image as `FLY_BASE_IMAGE`, which `qm sandbox publish` pins by digest. On a scope's next -provision, `FlySandbox.ensureMachine` compares the machine's own image reference to the target -and, if they differ, recreates the machine **keeping its volume** (`coldStart=false`, home -intact — no restore needed). A machine that predates volumes (no `/root` mount) is -migrated once: it's recreated with a fresh volume as a cold start, so the orchestrator -restores the object-store backup into it. Because that backup **excludes `~/.aws`** (see -below), the agent re-auths (`aws sso login`) once after the legacy migration. - -The Python venv is **baked into the rootfs at `/opt/agent-venv`** (on PATH), so it's present -the instant the machine boots — no per-scope bootstrap. It lives at `/opt`, not `/root`, -because `/root` is the per-scope volume mount that would SHADOW anything baked under it. This -removes the old first-provision `python -m venv` + network pip step (~5-15s on every cold -start) and keeps the venv off the home-volume backup. Trade-off: `pip install`s land in the -rootfs venv, so they reset when a scope's machine is recreated (image-version bump / reap). - -## Backup exclusions - -Fly's Agent Computer backup follows the Hermes S3-sync lesson: it excludes `.aws/*` -so stale AWS credential caches cannot shadow the platform role after restore. It also -excludes reproducible/noisy runtime caches (`__pycache__`, `.cache`, and any user-created -`~/venv`) that can make a resident-home export too large for provider APIs. (The agent's own -Python venv now lives in the rootfs at `/opt/agent-venv`, so it isn't in the home backup at all.) -Resident tool caches (a CLI's `~/./` state) can be backed up. Export/import is -batched as one tar stream per area (`workspace`, `home`) instead of one Fly Machines -API exec per file, which keeps backup viable after the agent creates many files. The -backup is durable **by default**: it is written to S3 when `SNAPSHOT_STORE=s3` is -configured (cross-machine), otherwise to the core's local persistent volume -(`$DATA_DIR/blobs`, single-machine durable) — it is never left un-persisted, because it -is the only durable record of agent files on this backend (the VM disk is canonical and -`snapshotWritable` is a no-op, so the workspace-store never sees them). The orchestrator -restores a backup only onto a freshly-created persistent computer, then writes a new -backup after the turn; warm VMs are left as the source of truth to avoid clobbering newer -resident disk state with an older snapshot. This backup also powers the admin **files** -view (`GET /v1/admin/files`), which reads it so agent-written files appear in the -dashboard even though they live on the VM, not in the workspace-store. - -## Turn env - -Fly drops host-proxy routing env (`http_proxy`, `HTTPS_PROXY`, etc.) when proxy routing is -not configured. It keeps other non-secret turn env (e.g. connector `VAULT_TOKEN_` -materialized for the acting user). Resident CLIs authenticate as the -box's resident machine identity — no acting-user claim is injected. - -## Egress - -Dangerous posture permits direct outbound network access. Auto forces traffic through the -audited proxy and blocks private/metadata destinations unless the admin explicitly allows a -host. Strict does not provision a sandbox. Configure the proxy as described in the deployment -guide before using Auto on Fly. - -## Build & deploy the base image (once) - -```bash -brew install flyctl -fly auth login -export FLY_SANDBOX_APP_NAME= -fly apps create "$FLY_SANDBOX_APP_NAME" --org -npm run deploy:fly-image -``` - -The base image keeps a minimal generic toolset (the coding-agent CLIs and AWS CLI v2; -the optional agentic browser engine is build-gated in `fly/Dockerfile`). -Deployment-specific tools are NOT baked here — a deployment stacks them on top via its -sandbox layer (`qm sandbox build` over `/sandbox/`). Anything else the agent needs is installed on the -**resident disk** of the persistent microVM, which survives across turns: optional CLIs -such as `glab` or X tooling are installed residently the first time they're needed, -exactly as on a real laptop. - -Use `npm run deploy:fly-image` rather than bare `fly deploy`: this sandbox app is -exec-only, and bare deploy creates default launch machines that are not used by -`FlySandbox`. - -Sandbox machines run `linux/amd64` only. `npm run deploy:fly-image` builds on Fly's -remote amd64 builder, so it works unchanged from arm64 (Apple Silicon) hosts, where a -local `docker build` produces an arm64 image the machines reject and -`--platform linux/amd64` under qemu emulation is slow and unreliable. -`scripts/local-sandbox-build.sh` follows the same rule: it uses the remote builder when -`FLY_SANDBOX_APP_NAME` is set and otherwise builds locally with -`--platform linux/amd64`. - -## Configure the core - -```bash -FLY_API_TOKEN="$(fly tokens create deploy -a "$FLY_SANDBOX_APP_NAME")" \ -FLY_SANDBOX_APP_NAME="$FLY_SANDBOX_APP_NAME" \ -FLY_BASE_IMAGE="registry.fly.io/$FLY_SANDBOX_APP_NAME@sha256:" \ -FLY_REGION=sjc \ -npm start -``` - -Other knobs: `FLY_CPU_KIND` (shared), `FLY_CPUS` (1), `FLY_MEMORY_MB` (512), -`FLY_AUTO_SUSPEND` (1 — suspend after each turn; ~½s resume), `SANDBOX_TIMEOUT_SEC` -(120 — the sandbox's bare per-command backstop; reached only on a standalone/misconfigured -path, since the orchestrator now always passes an explicit per-command timeout). - -**Per-command execute timeout.** Each `execute` command has a wall-clock cap (exit 124 on -kill). The agent sets it per command via the tool's `timeout_seconds` param; if it doesn't, the -command falls to the configured default. Knobs (orchestrator → tool context): -`EXEC_TIMEOUT_DEFAULT_SEC` (120 — covers an unanticipated moderately-long command: -npm install / tsc / a test run) and `EXEC_TIMEOUT_MAX_SEC` (300 — the hard ceiling the agent's -`timeout_seconds` is clamped to, so one session can't starve others; work beyond it should use -background execution / be broken into shorter steps). Resolution order: agent param > default > -sandbox backstop. - -> **Running the core itself ON Fly?** Fly injects `FLY_APP_NAME` at runtime as the -> _core's own_ app name, which would clobber the sandbox target. Set -> `FLY_SANDBOX_APP_NAME=` instead (wiring prefers it; see `src/wiring.ts`). -> `FLY_REGION` is likewise injected by Fly, so you can omit it on-Fly. -> Machines are named by scope (`personal-u1`, …); a scope reuses its machine across turns. - -Resident machine auth env is installed only when a Fly machine is first created. Set -new-machine env with the explicit `FLY_RESIDENT_ENV_` prefix, for example -`FLY_RESIDENT_ENV_AWS_ACCESS_KEY_ID`, `FLY_RESIDENT_ENV_AWS_SECRET_ACCESS_KEY`, -`FLY_RESIDENT_ENV_AWS_SESSION_TOKEN`, and `FLY_RESIDENT_ENV_AWS_DEFAULT_REGION`. -These are machine credentials for native CLIs such as `aws`, not egress proxy tokens. -For resident X tooling use the same prefix for native tool env, for example -`FLY_RESIDENT_ENV_X_BEARER_TOKEN` for `x-api`. - -## Smoke test - -```bash -FLY_API_TOKEN="$(fly tokens create deploy -a "$FLY_SANDBOX_APP_NAME")" npm run smoke:fly -``` - -For image-resident X helper readiness: - -```bash -FLY_API_TOKEN=... npm run smoke:x -``` - -This verifies `x-api` is on PATH and reports `missing_auth=auth_missing` when no -resident X token is installed. Add `X_SMOKE_REQUIRE_AUTH=1` after configuring -`X_BEARER_TOKEN` / `X_ACCESS_TOKEN`, or `X_SMOKE_REQUIRE_FIREHOSE=1` when a vendored -`x-firehose` binary should be present. - -The smoke test uses a timestamped personal smoke-test scope, writes -workspace and resident-home state, backs it up, deletes the Fly machine, recreates it, -restores the backup, verifies `.aws/*` stayed excluded, then deletes the smoke -machine. Add `SNAPSHOT_STORE=s3 S3_BUCKET=...` to exercise the real S3 object store; -without those vars it uses the same backup-store code over an in-memory blob store. - -For GitHub/GitLab resident CLI readiness: - -```bash -FLY_API_TOKEN=... npm run smoke:git-cli -``` - -This verifies `git` and `gh` are on PATH in a resident Fly computer and reports -whether `glab` is available. It also runs `gh auth status` and reports a sanitized -status (`ok`, `auth_missing`, `host_unreachable`, or `auth_error`) without printing -command output. If `glab` is available or required, it does the same for -`glab auth status`; otherwise GitLab auth is reported as `skipped`. Add -`GIT_CLI_SMOKE_REQUIRE_GLAB=1` when GitLab is meant to be supported by the current -image, `GIT_CLI_SMOKE_REQUIRE_GH_AUTH=1` after running `gh auth login` on the -resident computer, or `GIT_CLI_SMOKE_REQUIRE_GLAB_AUTH=1` after `glab auth login`. -A synthetic actor is destroyed by default; set -`GIT_CLI_SMOKE_ACTOR_ID=` when testing an existing resident computer. +The Dockerfile and Fly configuration in this directory remain inputs to the +local contributor sandbox build. They do not publish or select a Fly +deployment's Sprites runtime. diff --git a/package-lock.json b/package-lock.json index e2e634606..85533ad15 100644 --- a/package-lock.json +++ b/package-lock.json @@ -60,7 +60,7 @@ }, "cli": { "name": "@yc-software/qm", - "version": "0.1.0", + "version": "0.1.7", "dev": true, "license": "MIT", "bin": { diff --git a/package.json b/package.json index 6b33db575..f572ec098 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,6 @@ "smoke:local-sandbox": "node scripts/local-sandbox-smoke.ts", "build:aws-image": "node scripts/aws-build-sandbox-image.ts", "sandbox:local:build": "bash scripts/local-sandbox-build.sh", - "deploy:fly-image": "flyctl deploy --remote-only --build-only --push --image-label latest --app \"${FLY_SANDBOX_APP_NAME:?Set FLY_SANDBOX_APP_NAME to your operator-owned app}\" -c fly/fly.toml --dockerfile fly/Dockerfile . --yes", "test:pg": "node --test --test-concurrency=1 test/postgres-store.test.ts test/postgres-grant-store.test.ts test/postgres-admin-grants.test.ts test/postgres-file-artifact-store.test.ts test/postgres-directory-store.test.ts test/postgres-map.test.ts test/cron-queue.test.ts test/postgres-metrics-sink.test.ts test/postgres-error-log.test.ts test/postgres-audit-log.test.ts test/postgres-budget.test.ts test/postgres-rate-limiter.test.ts test/postgres-credential-usage-sink.test.ts test/postgres-config-store.test.ts test/postgres-delivery-store.test.ts test/postgres-replay-dedupe.test.ts test/postgres-egress-audit-sink.test.ts test/postgres-memory-service.test.ts test/run-signal-store.test.ts test/postgres-run-activity-store.test.ts test/leader-lease.test.ts test/advisory-lock.test.ts test/persistence-init-retry.test.ts test/migrate-principals.test.ts test/postgres-surface-cache.test.ts test/postgres-instance-registry.test.ts", "livetest": "node scripts/api-livetest.ts", "bench:memory": "node --env-file-if-exists=.env scripts/memory-bench.ts", diff --git a/plugins/admin/public/index.html b/plugins/admin/public/index.html index e7c0367b0..d06d4996a 100644 --- a/plugins/admin/public/index.html +++ b/plugins/admin/public/index.html @@ -2905,6 +2905,30 @@ text-overflow: ellipsis; white-space: nowrap; } + .environment-notice { + display: flex; + align-items: center; + justify-content: space-between; + gap: 18px; + margin: 0 0 18px; + padding: 14px 16px; + border: 1px solid color-mix(in srgb, var(--warn) 42%, var(--border)); + border-radius: 10px; + background: color-mix(in srgb, var(--warn) 8%, var(--surface)); + } + .environment-notice strong, + .environment-notice p { + display: block; + margin: 0; + } + .environment-notice p { + margin-top: 3px; + color: var(--muted); + font-size: 12px; + } + .environment-notice button { + flex: none; + } .governance-overview { margin: 0 0 22px; padding: 18px 20px; @@ -3987,6 +4011,13 @@

Governance

ScopeOrganization +

Effective state

@@ -5717,6 +5748,7 @@

Confirm governance change

}); let scopeDir = null; + let environmentDir = []; let scopeDirNote = "Loading scopes…"; async function loadScopeDirectory() { const r = await api("GET", "/api/scopes"); @@ -5728,6 +5760,7 @@

Confirm governance change

} viewLoadedAt.history = Date.now(); scopeDir = r.data.scopes || []; + environmentDir = r.data.environments || []; if (SCOPED.has(view) && !urlToState().session) { const memoryEditor = view === "memory" && !(orgWideView() && urlToState().mem !== "edit"); @@ -6244,6 +6277,19 @@

Confirm governance change

); } window.addEventListener("scroll", syncGovernanceSectionNav, { passive: true }); + function renderEnvironmentNotice(data) { + const notice = $("environment-notice"); + const attachment = data?.environmentAttachment; + notice.classList.toggle("hidden", !attachment); + if (!attachment) return; + const name = attachment.environmentName || shortName(attachment.environmentId); + $("environment-notice-title").textContent = "Uses named environment " + name; + $("environment-notice-detail").textContent = + "Computer files and working memory resolve to this environment. Governance and conversation history remain scoped here."; + $("environment-notice-open").textContent = "Open " + name; + $("environment-notice-open").onclick = () => + go({ view: "governance", scope: attachment.environmentId, session: null, page: 1 }); + } let governanceReq = 0; async function loadScope() { const requestedScope = scope; @@ -6259,6 +6305,7 @@

Confirm governance change

); return; } + renderEnvironmentNotice(r.data); renderGovernanceOverview(r.data); syncGovernanceSectionNav(); loadedCommandPolicyPresent = r.data.commandPolicy != null; @@ -11451,6 +11498,21 @@

Confirm governance change

actions: [sortControl], }); const activityTime = (s) => (scopeSort === "human" ? s.lastConversationActivity || 0 : s.lastActivity || 0); + if (environmentDir.length) { + const environments = denseList( + environmentDir, + (environment) => ({ + name: environment.name || shortName(environment.id), + preview: plural(environment.attachedScopes?.length || 0, "attached scope"), + href: stateToUrl({ view: "history", scope: environment.id, historyKind }), + }), + (environment) => selectScope(environment.id), + "No named environments.", + ); + root.appendChild( + dataCard("Named environments", "Named computers and working memory that scopes can share.", environments), + ); + } const t = denseList( activeRows, (s) => { diff --git a/plugins/admin/test/environments.test.ts b/plugins/admin/test/environments.test.ts new file mode 100644 index 000000000..3ec8c9fb8 --- /dev/null +++ b/plugins/admin/test/environments.test.ts @@ -0,0 +1,13 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +const html = readFileSync(join(import.meta.dirname, "../public/index.html"), "utf8"); + +test("the admin UI lists named environments and links attachment warnings", () => { + assert.match(html, /Named environments/); + assert.match(html, /id="environment-notice"/); + assert.match(html, /Uses named environment/); + assert.match(html, /scope: attachment\.environmentId/); +}); diff --git a/plugins/web-ui/src/chat.ts b/plugins/web-ui/src/chat.ts index de3ec466d..35db6f6aa 100644 --- a/plugins/web-ui/src/chat.ts +++ b/plugins/web-ui/src/chat.ts @@ -17,7 +17,6 @@ import { FileText, Files, GitFork, - Hash, Maximize2, Paperclip, Pencil, @@ -27,7 +26,6 @@ import { Rocket, ScrollText, Terminal, - Users, Wrench, type IconNode, } from "lucide"; @@ -52,7 +50,6 @@ import { makeOpenerStreamFn, makeRunResumeStreamFn, runApprovalTurn, - sharedContextLabel, TAIL_TURNS, type ApprovalDecision, type AssistantWork, @@ -84,7 +81,9 @@ import { harnessSupportsFastMode, } from "./model-options"; import { browserRenderableImage, formatBytes, icon, relTime } from "./ui"; -import { adminSessionLogUrl, appState, can, renderSidebarTop, syncUrlFromState } from "./shell"; +import { adminSessionLogUrl, appState, can, renderSidebarTop, switchView, syncUrlFromState } from "./shell"; +import { contextsState, scopeTitle } from "./contexts"; +import { openProjectPage, scopeCronCount, sessionTopbarTpl, setScopedSession } from "./session-scope"; import { addPendingSession, dropPendingSession, @@ -1024,7 +1023,7 @@ export function createChatSurface(
` : nothing } - ${contextBanner()} + ${glanceTier ? nothing : sessionTopbar(agent)} ${ glanceTier ? paneGlance(agent, messages, glanceTier) @@ -1086,16 +1085,40 @@ export function createChatSurface( return null; } - function contextBanner(): TemplateResult | typeof nothing { - const label = sharedContextLabel(chatState.scopeId, chatState.contextName); - if (!label) return nothing; - const glyph = chatState.scopeId?.startsWith("group:") ? Users : Hash; - return html`
- ${icon(glyph, 13)}${label} context -
`; + function pillState(needsYou: boolean, working: boolean): "needs-you" | "working" | null { + if (needsYou) return "needs-you"; + return working ? "working" : null; + } + + function sessionTopbar(agent: Agent): TemplateResult { + const scope = chatState.scopeId; + const session = sessionsState.list.find((s) => + chatState.sessionId + ? s.id === chatState.sessionId + : Boolean(chatState.threadRef) && s.threadRef === chatState.threadRef, + ); + const title = session?.title?.trim() || "New chat"; + const crumb = scope && !scope.startsWith("personal:") ? scopeTitle(scope, chatState.contextName) : null; + const needsYou = activePendingApprovals().length > 0; + const working = !needsYou && (agent.state.isStreaming || chatState.resolvingApprovals.size > 0); + return sessionTopbarTpl({ + crumb, + title, + onCrumb: crumb && scope ? () => openProjectPage(scope) : null, + pill: pillState(needsYou, working), + cronCount: scope ? scopeCronCount(scope, () => drawActiveChat()) : null, + onTool: (tool) => { + setScopedSession({ + scopeId: scope ?? "", + sessionId: chatState.sessionId, + threadRef: chatState.threadRef, + title, + crumb, + }); + if (scope && tool !== "memory") contextsState.selected = scope; + switchView(tool); + }, + }); } function chatHeader(title: string | TemplateResult, detail: string, readOnly: boolean): TemplateResult { @@ -1775,9 +1798,20 @@ export function createChatSurface( } function workSeconds(work: WorkBlock): number { - if (work.startedAt == null) return 0; - const end = work.finishedAt ?? Date.now(); - return Math.max(0, Math.round((end - work.startedAt) / 1000)); + const times = work.activity.map((a) => a.createdAt).filter((t) => typeof t === "number" && t > 0); + const start = work.startedAt ?? (times.length ? Math.min(...times) : null); + if (start == null) return 0; + const live = work.status === "thinking" || work.status === "working"; + let end = work.finishedAt; + if (end == null) { + if (live) end = Date.now(); + else end = times.length ? Math.max(...times, start) : start; + } + return Math.max(0, Math.round((end - start) / 1000)); + } + + function workedLabel(prefix: string, secs: number): string { + return secs > 0 ? `${prefix} for ${secs}s` : prefix; } function usedToolsSuffix(work: WorkBlock): string { @@ -1789,7 +1823,7 @@ export function createChatSurface( if (work.stale && (work.status === "thinking" || work.status === "working")) return "Interrupted — resuming…"; if (work.status === "thinking") return "Thinking"; const secs = workSeconds(work); - return work.status === "working" ? `Working for ${secs}s` : `Worked for ${secs}s`; + return work.status === "working" ? `Working for ${secs}s` : workedLabel("Worked", secs); } function workBlock(work: WorkBlock, isStreaming: boolean): TemplateResult { @@ -1827,7 +1861,11 @@ export function createChatSurface( }; for (const it of timeline) { const demoted = it.kind === "text" && (it.activity.payload as { demoted?: boolean } | null)?.demoted === true; - if (it.kind === "text" && !demoted) { + // Closing self-logs after a successful surface post are bookkeeping, not + // another piece of visible work. Keeping them in the transcript is useful + // for audit/replay, but rendering them creates an empty "Worked" fold. + if (demoted) continue; + if (it.kind === "text") { flushSeg(); const text = ((it.activity.payload as { text?: string } | null)?.text ?? "").trim(); if (text) parts.push(html`
${markdown(text)}
`); @@ -1836,14 +1874,15 @@ export function createChatSurface( } } flushSeg(); - return html`
${parts}
`; + return parts.length ? html`
${parts}
` : html``; } function segmentSummaryLabel(items: TimelineItem[], work: WorkBlock): string { const tools = items.filter((it) => it.kind === "tool").length; if (tools > 0) return `${tools} tool call${tools === 1 ? "" : "s"}`; const secs = workSeconds(work); - return work.status === "failed" ? `Failed after ${secs}s` : `Worked for ${secs}s`; + if (work.status === "failed") return secs > 0 ? `Failed after ${secs}s` : "Failed"; + return workedLabel("Worked", secs); } function approvalSummaryView(a: PendingApproval, expanded = false): TemplateResult { diff --git a/plugins/web-ui/src/contexts.ts b/plugins/web-ui/src/contexts.ts index ec42aeff4..e99b52d04 100644 --- a/plugins/web-ui/src/contexts.ts +++ b/plugins/web-ui/src/contexts.ts @@ -329,14 +329,32 @@ function gridTpl(): TemplateResult { Boolean(context.sessionCount)) ); }; - const rank = (context: CoreContext) => { - if (context.kind === "personal") return 0; - return context.project ? 1 : 2; + const projects = contextsState.list.filter(matches); + const groupOf = (context: CoreContext) => { + if (context.kind === "personal") return "personal"; + return context.project ? "web" : "slack"; }; - const projects = contextsState.list.filter(matches).sort((a, b) => rank(a) - rank(b)); + const groups = [ + { key: "personal", label: "Personal" }, + { key: "web", label: "Web" }, + { key: "slack", label: "Slack" }, + ] + .map((g) => ({ ...g, items: projects.filter((context) => groupOf(context) === g.key) })) + .filter((g) => g.items.length > 0); const projectsFiltered = Boolean(q); let projectList: TemplateResult | typeof nothing = nothing; - if (projects.length) projectList = html`
${projects.map(contextCard)}
`; + if (projects.length) + projectList = html`
+ ${groups.map( + (g) => + html`
+
+ ${g.label} · ${g.items.length} +
+ ${g.items.map(contextRow)} +
`, + )} +
`; else if (!contextsLoading) { projectList = html`
${projectsFiltered ? "No projects match your search." : "No projects yet."} @@ -399,22 +417,18 @@ function gridTpl(): TemplateResult { `; } -function contextCard(c: CoreContext): TemplateResult { +function contextRow(c: CoreContext): TemplateResult { const { title, sub, glyph } = contextMeta(c); const count = c.sessionCount === 1 ? "1 conversation" : `${c.sessionCount} conversations`; - let access = "shared"; - if (c.project && isProjectOwner(c)) access = "owned"; - else if (c.kind === "personal") access = "private"; + const meta = [c.project ? sub : "", count, c.lastActivityAt ? `active ${relTime(c.lastActivityAt)}` : ""] + .filter(Boolean) + .join(" · "); return html` - `; } diff --git a/plugins/web-ui/src/crons.ts b/plugins/web-ui/src/crons.ts index db13191b3..fd01f0536 100644 --- a/plugins/web-ui/src/crons.ts +++ b/plugins/web-ui/src/crons.ts @@ -4,7 +4,8 @@ import { api } from "./core-bridge"; import { errMessage } from "../../chassis/src/errors"; import { icon } from "./ui"; import { listBackLink, listPageTpl } from "./list-page"; -import { ensureContexts, scopeChip } from "./contexts"; +import { contextsState, ensureContexts, scopeChip } from "./contexts"; +import { scopedSession, scopedViewTopbar } from "./session-scope"; import { appState } from "./shell"; import { mainConversation } from "./conversations"; import { deepLinkPath, isPlainLeftClick, UI_BASE } from "./deep-link"; @@ -185,6 +186,11 @@ function cronStatusText(c: CronView): string { export async function renderCronsPage(): Promise { if (appState.currentView !== "crons") return; + if (scopedSession.active) cronsScope = scopedSession.active.scopeId; + else if (contextsState.selected) { + cronsScope = contextsState.selected; + contextsState.selected = null; + } await ensureContexts(); drawCronsPage(); const loaded = await refreshCrons({ showLoading: cronList.length === 0 && visibleCronList.length === 0 }); @@ -252,14 +258,19 @@ function drawCronsPage(): void { if (cronsNotice) empty = cronsNotice; else if (cronsLoading && cronList.length === 0 && visibleCronList.length === 0) empty = "Loading crons…"; else if (cronsScope) empty = "No crons in this context."; + const scoped = Boolean(scopedSession.active); + cronsPageHost.classList.toggle("scoped-view", scoped); render( - listPageTpl({ + html`${scopedViewTopbar("crons", drawCronsPage)} + ${listPageTpl({ title: "Crons", scope: cronsScope, - onScope: (s) => { - cronsScope = s; - drawCronsPage(); - }, + onScope: scoped + ? undefined + : (s) => { + cronsScope = s; + drawCronsPage(); + }, onRefresh: () => { cronRuns.clear(); void renderCronsPage(); @@ -275,7 +286,7 @@ function drawCronsPage(): void { }, rows, empty, - }), + })}`, cronsPageHost, ); } diff --git a/plugins/web-ui/src/files.ts b/plugins/web-ui/src/files.ts index 2fc996b90..d3712ce01 100644 --- a/plugins/web-ui/src/files.ts +++ b/plugins/web-ui/src/files.ts @@ -6,6 +6,7 @@ import { browserRenderableImage, fieldSelect, formatBytes, icon, relTime } from import { contextsState, ensureContexts, personalScopeId, scopeChip, scopeFilterControl } from "./contexts"; import { appState } from "./shell"; import { fileListNeedsAllPages } from "./file-list"; +import { scopedSession, scopedViewTopbar } from "./session-scope"; interface FileItem { id: string; @@ -93,20 +94,27 @@ function drawFiles(loading = false): void { else if (filesUploading) dropLabel = "Uploading…"; const status = filesNotice || (loading && !fileRows.length ? "Loading files…" : ""); const uploadTarget = filesScope ?? personalScopeId(); + const scoped = Boolean(scopedSession.active); + filesHost.classList.toggle("scoped-view", scoped); render( html` + ${scopedViewTopbar("files", drawFiles)}

Files

Files created, uploaded, or shared with you
- ${scopeFilterControl(filesScope, (s) => { - filesScope = s; - fileRows = []; - filesNextCursor = null; - void loadFiles(appState.viewRenderSeq); - })}
@@ -386,7 +394,14 @@ async function loadFiles(seq: number): Promise { export async function renderFiles(): Promise { if (appState.currentView !== "files") return; - if (contextsState.selected) { + if (scopedSession.active) { + if (filesScope !== scopedSession.active.scopeId) { + filesScope = scopedSession.active.scopeId; + fileRows = []; + filesNextCursor = null; + } + contextsState.selected = null; + } else if (contextsState.selected) { filesScope = contextsState.selected; fileRows = []; filesNextCursor = null; diff --git a/plugins/web-ui/src/memory.ts b/plugins/web-ui/src/memory.ts index cddef5a2d..21efbc74c 100644 --- a/plugins/web-ui/src/memory.ts +++ b/plugins/web-ui/src/memory.ts @@ -4,6 +4,7 @@ import { api, ApiError } from "./core-bridge"; import { errMessage } from "../../chassis/src/errors"; import { icon } from "./ui"; import { appState, replacePanePreservingFocus } from "./shell"; +import { scopedSession, scopedViewTopbar } from "./session-scope"; interface RevisionRow { revision: string; @@ -64,9 +65,10 @@ function drawMemory(loading = false): void { (fact) => !search || fact.text.toLowerCase().includes(search.toLowerCase()), ); const host = document.createElement("div"); - host.className = "pane"; + host.className = scopedSession.active ? "pane scoped-view" : "pane"; render( html` + ${scopedViewTopbar("memory", () => drawMemory())}

Memory

diff --git a/plugins/web-ui/src/session-scope.ts b/plugins/web-ui/src/session-scope.ts new file mode 100644 index 000000000..fc73e7592 --- /dev/null +++ b/plugins/web-ui/src/session-scope.ts @@ -0,0 +1,165 @@ +import { html, nothing, type TemplateResult } from "lit"; +import { Brain, Clock3, Files } from "lucide"; +import { api } from "./core-bridge"; +import { icon } from "./ui"; + +/** A session's context carried into the crons/files/memory views so the whole + * view stays scoped to that project and keeps the session top bar. */ +export interface ScopedSessionInfo { + scopeId: string; + sessionId: string | null; + threadRef: string | null; + title: string; + crumb: string | null; +} + +export const scopedSession: { active: ScopedSessionInfo | null } = { active: null }; + +export function setScopedSession(info: ScopedSessionInfo | null): void { + scopedSession.active = info; +} + +interface CronLite { + id: string; + ownerScopeId: string; + enabled: boolean; + archived?: boolean; +} + +const cronCountCache = new Map(); +const cronCountInFlight = new Set(); + +/** Cached count of enabled crons owned by a scope; kicks off a refresh and + * calls onReady when a fresh count lands. */ +export function scopeCronCount(scope: string, onReady: () => void): number | null { + const hit = cronCountCache.get(scope); + if (hit && Date.now() - hit.at < 60_000) return hit.count; + if (!cronCountInFlight.has(scope)) { + cronCountInFlight.add(scope); + void api<{ crons?: CronLite[]; visible?: CronLite[] }>("/api/crons") + .then((r) => { + const seen = new Set(); + let count = 0; + for (const c of [...(r.crons ?? []), ...(r.visible ?? [])]) { + if (seen.has(c.id)) continue; + seen.add(c.id); + if (c.ownerScopeId === scope && c.enabled && !c.archived) count++; + } + cronCountCache.set(scope, { count, at: Date.now() }); + }) + .catch(() => cronCountCache.set(scope, { count: hit?.count ?? 0, at: Date.now() })) + .finally(() => { + cronCountInFlight.delete(scope); + onReady(); + }); + } + return hit?.count ?? null; +} + +export type SessionTool = "crons" | "files" | "memory"; + +export interface SessionTopbarOpts { + crumb: string | null; + title: string; + pill?: "working" | "needs-you" | null; + activeTool?: SessionTool | null; + cronCount?: number | null; + onTitle?: (() => void) | null; + onCrumb?: (() => void) | null; + onTool: (tool: SessionTool) => void; +} + +export function sessionTopbarTpl(o: SessionTopbarOpts): TemplateResult { + const crumbTpl = ((): TemplateResult | typeof nothing => { + if (!o.crumb) return nothing; + if (!o.onCrumb) return html`${o.crumb}/`; + return html`/`; + })(); + const heading = html` + ${crumbTpl} + ${o.title} + ${ + o.pill + ? html`${o.pill === "needs-you" ? "needs you" : "working"}` + : nothing + } + `; + const headingTitle = o.crumb + ? `This chat runs in the ${o.crumb} context — the agent works with that context's files and memory, separate from your personal context.` + : o.title; + const tool = (t: SessionTool, glyph: Parameters[0], label: string, hint: string) => html` + + `; + const cronLabel = o.cronCount ? `${o.cronCount} ${o.cronCount === 1 ? "cron" : "crons"}` : "Crons"; + return html` +
+ ${ + o.onTitle + ? html`` + : html`
${heading}
` + } +
+ ${tool("crons", Clock3, cronLabel, "Crons in this context")} + ${tool("files", Files, "Files", "Files in this context")} ${tool("memory", Brain, "Memory", "Memory")} +
+
+ `; +} + +export function openProjectPage(scopeId: string): void { + setScopedSession(null); + void import("./contexts").then(({ openProjectDetail }) => openProjectDetail(scopeId)); +} + +/** Top bar for the scoped crons/files/memory views: same bar, title links back + * to the session, tools swap views while keeping the scope. */ +export function scopedViewTopbar(current: SessionTool, redraw: () => void): TemplateResult | typeof nothing { + const active = scopedSession.active; + if (!active) return nothing; + return sessionTopbarTpl({ + crumb: active.crumb, + title: active.title, + onCrumb: active.crumb ? () => openProjectPage(active.scopeId) : null, + activeTool: current, + cronCount: scopeCronCount(active.scopeId, redraw), + onTitle: () => { + setScopedSession(null); + void Promise.all([import("./shell"), import("./sessions")]).then( + ([{ appState, renderSidebarTop }, { sessionsState, openSession }]) => { + const s = sessionsState.list.find( + (row) => (active.sessionId && row.id === active.sessionId) || row.threadRef === active.threadRef, + ); + if (!s) return; + appState.currentView = "chats"; + renderSidebarTop(); + void openSession(s); + }, + ); + }, + onTool: (t) => { + if (t === current) return; + void import("./shell").then(({ switchView }) => switchView(t)); + }, + }); +} diff --git a/plugins/web-ui/src/sessions.ts b/plugins/web-ui/src/sessions.ts index 738e164f0..f601f4ac4 100644 --- a/plugins/web-ui/src/sessions.ts +++ b/plugins/web-ui/src/sessions.ts @@ -80,6 +80,7 @@ import { beginSessionDrag, endSessionDrag, notifySessionsChanged, + closeSessionSurfaces, sessionInCanvas, splitInterceptsOpen, splitState, @@ -1068,6 +1069,7 @@ async function commitRename(s: CoreSession): Promise { function setArchived(s: CoreSession, archived: boolean): void { sessionsState.openMenuId = null; + if (archived && s.id) closeSessionSurfaces(s.id); void persistSessionPatch(s.id, { archived }); } diff --git a/plugins/web-ui/src/shell.css b/plugins/web-ui/src/shell.css index e4f6fd67f..46e636b9f 100644 --- a/plugins/web-ui/src/shell.css +++ b/plugins/web-ui/src/shell.css @@ -1049,6 +1049,134 @@ a.chat-row-open { align-items: center; gap: 4px; } +.session-topbar { + gap: 12px; +} +.session-heading { + min-width: 0; + flex: 1; + display: flex; + align-items: center; + gap: 8px; + font-size: 14px; + line-height: 1.25; +} +.session-crumb { + color: var(--muted-foreground); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + flex: 0 3 auto; +} +.session-crumb-sep { + color: color-mix(in srgb, var(--muted-foreground) 65%, transparent); + flex: none; +} +.session-title { + font-weight: 650; + min-width: 0; + flex: 0 1 auto; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +.session-pill { + flex: none; + display: inline-flex; + align-items: center; + gap: 5px; + padding: 2px 9px; + border-radius: 999px; + font-size: 12px; + font-weight: 550; +} +.session-pill .pill-dot { + width: 6px; + height: 6px; + border-radius: 50%; + background: currentColor; +} +.session-pill.working { + color: color-mix(in srgb, #3b82f6 85%, var(--foreground)); + background: color-mix(in srgb, #3b82f6 12%, transparent); +} +.session-pill.needs-you { + color: color-mix(in srgb, #d97706 85%, var(--foreground)); + background: color-mix(in srgb, #f59e0b 14%, transparent); +} +.session-tools { + gap: 2px; + flex: none; +} +.session-tool { + display: inline-flex; + align-items: center; + gap: 6px; + height: 30px; + padding: 0 10px; + border: 0; + border-radius: 8px; + background: transparent; + color: var(--muted-foreground); + font-size: 12.5px; + font-weight: 550; + cursor: pointer; + white-space: nowrap; +} +.session-tool:hover { + background: color-mix(in srgb, var(--secondary) 70%, transparent); + color: var(--foreground); +} +.session-tool.active { + background: color-mix(in srgb, var(--secondary) 85%, transparent); + color: var(--foreground); +} +.session-heading.as-link { + border: 0; + background: transparent; + padding: 0; + cursor: pointer; + text-align: left; + color: inherit; + font: inherit; +} +.session-heading.as-link:hover .session-title { + text-decoration: underline; +} +.pane .session-topbar { + margin: -28px -28px 12px; + position: sticky; + top: -28px; + z-index: 5; +} +.scoped-view .pane-title { + font-size: 14px; + font-weight: 550; + color: var(--muted-foreground); +} +.scoped-view .pane-subtitle { + display: none; +} +.scoped-view .list-page-head, +.scoped-view .pane-head { + margin-bottom: 8px; +} +.session-crumb.as-link { + border: 0; + background: transparent; + padding: 0; + font: inherit; + cursor: pointer; + color: var(--muted-foreground); + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + flex: 0 3 auto; +} +.session-crumb.as-link:hover { + color: var(--foreground); + text-decoration: underline; +} .icon-btn { width: 34px; height: 34px; @@ -4139,6 +4267,13 @@ a.chat-row-open { height: calc(54px + var(--surface-safe-top)); padding: var(--surface-safe-top) max(10px, env(safe-area-inset-right)) 0 max(10px, env(safe-area-inset-left)); } + .session-tool span { + display: none; + } + .session-crumb, + .session-crumb-sep { + display: none; + } .message-stack { gap: 6px; } @@ -4383,38 +4518,69 @@ a.chat-row-open { background: var(--secondary); font-size: 13px; } -.project-grid { - grid-template-columns: repeat(auto-fill, minmax(240px, 1fr)); +.project-list { + width: min(960px, 100%); + margin: 0 auto; } -.project-empty { - max-width: 960px; - margin: 12px auto 0; - border: 1px dashed var(--border); - box-sizing: border-box; +.project-group { + margin-top: 22px; } -.project-other-label { - max-width: 960px; - margin: 28px auto 0; +.project-group-head { + padding: 0 6px 6px; + border-bottom: 1px solid var(--border); + color: var(--muted-foreground); + font-size: 11px; + font-weight: 700; + letter-spacing: 0.08em; + text-transform: uppercase; +} +.project-group-count { + font-weight: 500; + letter-spacing: normal; } -.context-card { +.context-row { + display: flex; + align-items: center; + gap: 10px; + width: 100%; + padding: 7px 6px; + border: 0; + border-bottom: 1px solid var(--border); + border-radius: 0; + background: transparent; + color: var(--foreground); font: inherit; text-align: left; cursor: pointer; - transition: - background 0.12s ease, - border-color 0.12s ease; + white-space: nowrap; } -.context-card:hover { +.context-row:hover { background: color-mix(in srgb, var(--background) 88%, var(--secondary)); - border-color: color-mix(in srgb, var(--foreground) 18%, var(--border)); } -.context-card .card-head { - align-items: center; - justify-content: flex-start; +.context-row-title { + overflow: hidden; + min-width: 0; + flex-shrink: 1; + text-overflow: ellipsis; + font-weight: 600; } -.context-card .card-head .badge { +.context-row-meta { + overflow: hidden; margin-left: auto; - flex-shrink: 0; + flex: none; + color: var(--muted-foreground); + font-size: 12.5px; + text-overflow: ellipsis; +} +.project-empty { + max-width: 960px; + margin: 12px auto 0; + border: 1px dashed var(--border); + box-sizing: border-box; +} +.project-other-label { + max-width: 960px; + margin: 28px auto 0; } .pane-subtitle { @@ -6630,6 +6796,11 @@ h3.ambient-field-label { color: var(--foreground); } +.split-canvas.session-dragging .dv-tabs-and-actions-container { + box-shadow: inset 0 0 0 2px color-mix(in srgb, var(--primary) 55%, var(--border)); + background: color-mix(in srgb, var(--primary) 6%, transparent); +} + .zone-top { left: 25%; right: 25%; diff --git a/plugins/web-ui/src/shell.ts b/plugins/web-ui/src/shell.ts index 1f29ae2b8..e004bdb61 100644 --- a/plugins/web-ui/src/shell.ts +++ b/plugins/web-ui/src/shell.ts @@ -58,6 +58,7 @@ import { } from "./sessions"; import { openCronById, renderCronsPage, resetActiveCron, routeCronsHistory } from "./crons"; import { renderFiles } from "./files"; +import { setScopedSession } from "./session-scope"; import { clearConnectorNotice, noteConnectorResult, renderConnectors, resetKeychainState } from "./connectors"; import { renderDeploys } from "./deploys"; import { renderMemory, resetMemoryState } from "./memory"; @@ -564,6 +565,7 @@ function onNavClick(e: Event): void { if (!isView(view)) return; if (e instanceof MouseEvent && !isPlainLeftClick(e)) return; e.preventDefault(); + setScopedSession(null); switchView(view); closeSidebarOnNarrowView(); } diff --git a/plugins/web-ui/src/split.ts b/plugins/web-ui/src/split.ts index 90b8fecd3..5c31b4cbc 100644 --- a/plugins/web-ui/src/split.ts +++ b/plugins/web-ui/src/split.ts @@ -152,6 +152,7 @@ function buildDock(): DockviewApi { guarded.add(group); group.model.onWillDrop(holdTileCap); } + if (sessionDrag) refreshSessionDrag(); persistSoon(); }); api.onDidActivePanelChange((e) => { @@ -361,6 +362,23 @@ export function sessionInCanvas(sessionId: string): boolean { return splitState.active && paneShowing(sessionId) !== null; } +/** Close any pane (and, outside the canvas, the main view) showing this session. */ +export function closeSessionSurfaces(sessionId: string): boolean { + if (!sessionId) return false; + if (splitState.active && dockApi) { + const showing = dockApi.panels.filter((p) => panelParams(p).sessionId === sessionId); + if (showing.length) { + closePanels(showing); + return true; + } + return false; + } + const conv = mainConversation(); + if (conv.state.sessionId !== sessionId) return false; + conv.newChat(); + return true; +} + export function splitInterceptsOpen(s: CoreSession): boolean { if (!splitState.active || appState.currentView !== "chats" || !s.id) return false; const target = splitState.focusedId ?? dockApi?.panels[0]?.id ?? ""; @@ -513,14 +531,23 @@ function drawToast(): void { export function beginSessionDrag(s: CoreSession): void { if (!s.id) return; sessionDrag = { sessionId: s.id, threadRef: s.threadRef }; - if (splitState.active) syncAllZones(); + if (splitState.active) refreshSessionDrag(); else showSingleDropOverlay(); } +function refreshSessionDrag(): void { + const drag = sessionDrag; + if (!drag) return; + const addsTab = !paneShowing(drag.sessionId) && (dockApi?.panels.length ?? 0) < MAX_PANES; + canvasHost?.classList.toggle("session-dragging", addsTab); + syncAllZones(); +} + export function endSessionDrag(): void { if (!sessionDrag) return; sessionDrag = null; hideSingleDropOverlay(); + canvasHost?.classList.remove("session-dragging"); if (splitState.active) syncAllZones(); } @@ -548,14 +575,33 @@ function zoneTpl(edge: DropEdge, label: string, onDrop: () => void): TemplateRes
`; } -function zonesTpl(act: (edge: DropEdge) => () => void): TemplateResult { +function splitZonesTpl(act: (edge: DropEdge) => () => void): TemplateResult { return html` - ${zoneTpl("center", "Open here", act("center"))} ${zoneTpl("left", "Split left", act("left"))} - ${zoneTpl("right", "Split right", act("right"))} ${zoneTpl("top", "Split up", act("top"))} - ${zoneTpl("bottom", "Split down", act("bottom"))} + ${zoneTpl("left", "Split left", act("left"))} ${zoneTpl("right", "Split right", act("right"))} + ${zoneTpl("top", "Split up", act("top"))} ${zoneTpl("bottom", "Split down", act("bottom"))} `; } +function zonesTpl(act: (edge: DropEdge) => () => void): TemplateResult { + return html`${zoneTpl("center", "Open here", act("center"))} ${splitZonesTpl(act)}`; +} + +function paneZonesTpl(paneId: string): TemplateResult | typeof nothing { + const drag = sessionDrag; + if (!drag || !dockApi) return nothing; + const act = paneZoneAct(paneId); + const showing = paneShowing(drag.sessionId); + if (showing) + return showing.id === paneId + ? zoneTpl("center", "Show here", () => { + endSessionDrag(); + focusPane(paneId); + }) + : nothing; + const canSplit = dockApi.panels.length < MAX_PANES && dockApi.groups.length < MAX_TILES; + return html`${zoneTpl("center", "Open here", act("center"))} ${canSplit ? splitZonesTpl(act) : nothing}`; +} + function paneZoneAct(paneId: string): (edge: DropEdge) => () => void { return (edge) => () => { const drag = sessionDrag; @@ -595,9 +641,12 @@ function showSingleDropOverlay(): void { } activateCanvas(current, { sessionId: drag.sessionId, threadRef: drag.threadRef }, edge); }; + const drag = sessionDrag; + const splittable = + Boolean(drag) && mainConversation().state.sessionId !== drag?.sessionId && currentChatParams() !== null; singleOverlay = document.createElement("div"); singleOverlay.className = "split-zones split-zones-single"; - render(zonesTpl(act), singleOverlay); + render(splittable ? zonesTpl(act) : zoneTpl("center", "Open here", act("center")), singleOverlay); appState.mainEl.appendChild(singleOverlay); } @@ -768,7 +817,7 @@ class PaneContent implements IContentRenderer { } syncZones(): void { - render(sessionDrag ? zonesTpl(paneZoneAct(this.panelId)) : nothing, this.zonesEl); + render(sessionDrag ? paneZonesTpl(this.panelId) : nothing, this.zonesEl); } dispose(): void { diff --git a/plugins/web-ui/test/archive-closes-pane.test.ts b/plugins/web-ui/test/archive-closes-pane.test.ts new file mode 100644 index 000000000..0d30f8478 --- /dev/null +++ b/plugins/web-ui/test/archive-closes-pane.test.ts @@ -0,0 +1,29 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +const read = (f: string): string => readFileSync(new URL(`../src/${f}`, import.meta.url), "utf8"); +const split = read("split.ts"); +const sessions = read("sessions.ts"); + +const fn = (src: string, name: string): string => { + const body = src.match(new RegExp(`^(?:export )?(?:async )?function ${name}\\([\\s\\S]*?\\n\\}`, "m"))?.[0] ?? ""; + assert.ok(body, `${name} not found`); + return body; +}; + +test("archiving a session closes any surface still showing it", () => { + const close = fn(split, "closeSessionSurfaces"); + assert.match(close, /dockApi\.panels\.filter\(\(p\) => panelParams\(p\)\.sessionId === sessionId\)/); + assert.match(close, /closePanels\(showing\)/, "open panes must be removed, and reconciled by closePanels"); + assert.match(close, /conv\.state\.sessionId !== sessionId\) return false;/, "leave other conversations alone"); + assert.match(close, /conv\.newChat\(\);/, "outside the canvas the main view drops the archived session"); + + const archived = fn(sessions, "setArchived"); + assert.match(archived, /if \(archived && s\.id\) closeSessionSurfaces\(s\.id\);/); + assert.ok( + archived.indexOf("closeSessionSurfaces") < archived.indexOf("persistSessionPatch"), + "close the surface before the patch round-trip so the UI reacts immediately", + ); + assert.doesNotMatch(archived, /!archived.*closeSessionSurfaces/s, "unarchiving must not close anything"); +}); diff --git a/plugins/web-ui/test/split-canvas-entry.test.ts b/plugins/web-ui/test/split-canvas-entry.test.ts index 6a3e92172..4009e1420 100644 --- a/plugins/web-ui/test/split-canvas-entry.test.ts +++ b/plugins/web-ui/test/split-canvas-entry.test.ts @@ -96,8 +96,8 @@ test("a strip drop lands where a dragged pane header would, not merely at the en test("the pane body no longer offers a tab zone", () => { assert.doesNotMatch(layout, /"tab"/, "DropEdge must drop the zone that no longer exists"); - const zones = fn(split, "zonesTpl"); - assert.doesNotMatch(zones, /tab/i); + const zones = fn(split, "zonesTpl") + fn(split, "splitZonesTpl"); + assert.doesNotMatch(zones, /"tab"/); assert.match(zones, /zoneTpl\("center", "Open here"/); for (const edge of ["left", "right", "top", "bottom"]) assert.match(zones, new RegExp(`zoneTpl\\("${edge}"`)); assert.doesNotMatch(css, /\.zone-tab \{/); diff --git a/plugins/web-ui/test/split-drop-legitimacy.test.ts b/plugins/web-ui/test/split-drop-legitimacy.test.ts new file mode 100644 index 000000000..19d00d339 --- /dev/null +++ b/plugins/web-ui/test/split-drop-legitimacy.test.ts @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +const css = readFileSync(new URL("../src/shell.css", import.meta.url), "utf8"); +const split = readFileSync(new URL("../src/split.ts", import.meta.url), "utf8"); + +test("a pane only offers the drops that will really happen", () => { + const zones = split.match(/^function paneZonesTpl\([\s\S]*?\n\}/m)?.[0] ?? ""; + assert.ok(zones, "paneZonesTpl not found"); + assert.match(zones, /paneShowing\(drag\.sessionId\)/, "a session already on the canvas gets no split/tab targets"); + assert.match( + zones, + /"Show here", \(\) => \{\n\s*endSessionDrag\(\);\n\s*focusPane\(paneId\);/, + "its own pane offers a target that really focuses it", + ); + assert.match(zones, /groups\.length < MAX_TILES/, "split targets vanish at the tile cap"); + assert.match(zones, /panels\.length < MAX_PANES/, "split targets also vanish at the pane ceiling"); + assert.match(split, /render\(sessionDrag \? paneZonesTpl\(this\.panelId\) : nothing, this\.zonesEl\);/); +}); + +test("the tab strips light up as targets from the moment the drag starts", () => { + const refresh = split.match(/^function refreshSessionDrag\([\s\S]*?\n\}/m)?.[0] ?? ""; + assert.ok(refresh, "refreshSessionDrag not found"); + assert.match( + refresh, + /!paneShowing\(drag\.sessionId\) && \(dockApi\?\.panels\.length \?\? 0\) < MAX_PANES/, + "but only when a strip drop would really add a tab", + ); + assert.match(refresh, /classList\.toggle\("session-dragging", addsTab\)/); + assert.match(split, /classList\.remove\("session-dragging"\)/); + assert.match(css, /\.split-canvas\.session-dragging \.dv-tabs-and-actions-container \{/); +}); + +test("targets and highlights stay honest when the layout changes mid-drag", () => { + assert.match( + split, + /if \(sessionDrag\) refreshSessionDrag\(\);\n\s*persistSoon\(\);/, + "a layout change during a drag recomputes the drop targets and strip highlight", + ); +}); + +test("the single-view overlay hides splits the drop cannot honor", () => { + const single = split.match(/^function showSingleDropOverlay\([\s\S]*?\n\}/m)?.[0] ?? ""; + assert.ok(single, "showSingleDropOverlay not found"); + assert.match( + single, + /currentChatParams\(\) !== null/, + "a dirty unsaved chat cannot seed a pane, so no split targets", + ); + assert.match(single, /state\.sessionId !== drag\?\.sessionId/, "dropping the chat onto itself cannot split"); + assert.match(single, /splittable \? zonesTpl\(act\) : zoneTpl\("center", "Open here", act\("center"\)\)/); +}); diff --git a/plugins/web-ui/test/split-layout.test.ts b/plugins/web-ui/test/split-layout.test.ts index 502e9c1af..6f30ac70e 100644 --- a/plugins/web-ui/test/split-layout.test.ts +++ b/plugins/web-ui/test/split-layout.test.ts @@ -43,6 +43,10 @@ test("a conversation can be dropped onto a pane's tab strip to become a tab", () const splitTs = readFileSync(new URL("../src/split.ts", import.meta.url), "utf8"); assert.match(splitTs, /e\.target === "tab" \|\| e\.target === "header_space"/, "no strip drop target"); assert.match(splitTs, /api\.onDidDrop\(\(e\) => \{[\s\S]*?tabIntoPane\(/, "the strip drop must join the pane"); - const edges = [...splitTs.matchAll(/zoneTpl\("([a-z]+)"/g)].map((m) => m[1]); - assert.deepEqual(edges, ["center", "left", "right", "top", "bottom"], "the body zones must not restate the strip"); + const edges = new Set([...splitTs.matchAll(/zoneTpl\("([a-z]+)"/g)].map((m) => m[1])); + assert.deepEqual( + edges, + new Set(["center", "left", "right", "top", "bottom"]), + "the body zones must not restate the strip", + ); }); diff --git a/plugins/web-ui/test/work-fold-source.test.ts b/plugins/web-ui/test/work-fold-source.test.ts index 11b6d3f49..3fbe7f0c1 100644 --- a/plugins/web-ui/test/work-fold-source.test.ts +++ b/plugins/web-ui/test/work-fold-source.test.ts @@ -6,7 +6,7 @@ const chat = readFileSync(new URL("../src/chat.ts", import.meta.url), "utf8"); const css = readFileSync(new URL("../src/shell.css", import.meta.url), "utf8"); test("finished turns render mid-turn text OUTSIDE the collapsed fold, at its place in the timeline", () => { - assert.match(chat, /if \(it\.kind === "text" && !demoted\) \{\s*\n\s*flushSeg\(\);/); + assert.match(chat, /if \(demoted\) continue;\s*\n\s*if \(it\.kind === "text"\) \{\s*\n\s*flushSeg\(\);/); assert.match(chat, /class="work-said"/); assert.match(chat, /
{ assert.match(css, /\.work-said \{[\s\S]{0,200}?color: var\(--foreground\);/); }); -test("a demoted post-delivery self-log stays folded as narration — never promoted to speech", () => { +test("a demoted post-delivery self-log remains auditable but is omitted from the UI", () => { const bridge = readFileSync(new URL("../src/core-bridge.ts", import.meta.url), "utf8"); assert.match(bridge, /payload: \{ text, demoted: true \}/); assert.match(chat, /demoted === true/); - assert.match(chat, /it\.kind === "text" && !demoted/); + assert.match(chat, /if \(demoted\) continue;/); + assert.match(chat, /return parts\.length \? .* : html``;/); }); test("the fold chevron rotates when a work-fold is open", () => { diff --git a/scripts/monitor-smoke.ts b/scripts/monitor-smoke.ts index 0ba165240..996e1e923 100644 --- a/scripts/monitor-smoke.ts +++ b/scripts/monitor-smoke.ts @@ -8,7 +8,7 @@ import { scopeId, type TurnRequest, type TurnResult } from "../src/types.ts"; import { createMemoryProcessRegistry } from "../src/processes/process-registry.ts"; import { createBackgroundBroker } from "../src/connectors/background-exec-broker.ts"; import { createMonitorStore } from "../src/monitors/monitor-store.ts"; -import { createMonitorBroker } from "../src/monitors/monitor-broker.ts"; +import { createMonitorBroker, readBackgroundOutputTail } from "../src/monitors/monitor-broker.ts"; import { createMonitorPoller } from "../src/monitors/monitor-poller.ts"; import { createDeliveryStore } from "../src/delivery/delivery-store.ts"; import { createIdempotencyStore } from "../src/idempotency/idempotency-store.ts"; @@ -64,9 +64,22 @@ const registry = createMemoryProcessRegistry(); const monitors = createMonitorStore(); const deliveries = createDeliveryStore(); const broker = createBackgroundBroker({ sandbox: sb, registry, scopeId: scope, pollMs: 4000, ttlMaxMs: 60 * 60_000 }); +let h: Awaited> | undefined; const monitorBroker = createMonitorBroker({ store: monitors, registry, + readOutputTail: async (processId, maxBytes) => { + if (!h) return { outputTail: "" }; + const handle = h; + return readBackgroundOutputTail(maxBytes, async (cursor, readMaxBytes) => { + const read = await broker.poll(handle, processId, { sinceCursor: cursor, maxBytes: readMaxBytes, waitMs: 0 }); + return { + chunks: read.chunks, + cursor: read.cursor, + ...(read.status.state === "exited" ? { exitCode: read.status.code } : {}), + }; + }); + }, scopeId: scope, owner, ownerScopeId: scope, @@ -97,7 +110,6 @@ const poller = createMonitorPoller({ }, }); -let h: Awaited> | undefined; try { console.log(`[${ts()}] provisioning sprite for`, scope, "…"); h = await sb.provision([{ scopeId: scope, mountPath: "", mode: "rw" }]); @@ -117,6 +129,7 @@ try { instructions: "Briefly tell the user how the diffusion-model download/generation is going.", sinceCursor: s.cursor, }); + if ("completed" in w) throw new Error(`job already ${w.registryStatus} before watch could be armed`); ok(!w.reattached, `watch armed (monitor ${w.monitorId}, expires ${new Date(w.expiresAt).toISOString()})`); console.log(`[${ts()}] polling every 10s until the exit wake (cap 35min) …`); diff --git a/src/api/app-sessions.ts b/src/api/app-sessions.ts index 6faaf7f23..4bf62a7f9 100644 --- a/src/api/app-sessions.ts +++ b/src/api/app-sessions.ts @@ -44,6 +44,7 @@ export function createSessionMethods( | "updateSession" | "regenerateTitle" | "spawnSession" + | "discardSession" | "forkSession" | "grant" | "revokeGrant" @@ -518,6 +519,22 @@ export function createSessionMethods( return create(); }, + async discardSession(sessionId, principalId) { + const session = await deps.sessions.get(sessionId); + if (!session) return false; + const mine = await deps.sessions.listByParticipant(principalId); + if (!mine.some((s) => s.id === sessionId)) return false; + if (!(await deps.sessions.deleteSessionIfEmpty(sessionId))) return false; + deps.auditLog.record({ + at: Date.now(), + principalId, + action: "session.discard", + resource: sessionId, + scopeLabel: session.scopeId, + }); + return true; + }, + async grant(g) { await deps.acl.grant(g, await artifactAuthor(g.ownerScopeId, g.ref)); deps.auditLog.record({ diff --git a/src/api/app-types.ts b/src/api/app-types.ts index 679567dc7..ae62386f3 100644 --- a/src/api/app-types.ts +++ b/src/api/app-types.ts @@ -281,6 +281,7 @@ export interface App { ): Promise; regenerateTitle(sessionId: string, principalId: string): Promise<{ title: string | null } | null>; spawnSession(principalId: string, opts: { scopeId: ScopeId; title?: string }): Promise<{ session: Session } | null>; + discardSession(sessionId: string, principalId: string): Promise; forkSession( sessionId: string, principalId: string, diff --git a/src/api/http.ts b/src/api/http.ts index 7a1bf5e0a..56f196e0f 100644 --- a/src/api/http.ts +++ b/src/api/http.ts @@ -7,6 +7,43 @@ export function sendJson(res: ServerResponse, status: number, body: unknown): vo res.end(data); } +export function contentTypeWithUtf8Charset(contentType: string): string { + let parameterStart = -1; + let inQuotes = false; + let escaped = false; + for (let i = 0; i < contentType.length; i += 1) { + const character = contentType[i]!; + if (escaped) { + escaped = false; + } else if (inQuotes && character === "\\") { + escaped = true; + } else if (character === '"') { + inQuotes = !inQuotes; + } else if (!inQuotes && character === ";") { + if (parameterStart >= 0 && /^\s*charset\s*=/i.test(contentType.slice(parameterStart, i))) return contentType; + parameterStart = i + 1; + } + } + if (parameterStart >= 0 && /^\s*charset\s*=/i.test(contentType.slice(parameterStart))) return contentType; + const mime = contentType.split(";", 1)[0]!.trim().toLowerCase(); + const textual = + mime.startsWith("text/") || + mime === "application/json" || + mime === "application/xml" || + mime === "application/yaml" || + mime === "application/x-yaml" || + mime === "application/javascript" || + mime === "application/x-javascript" || + mime === "application/graphql" || + mime === "application/sql" || + mime === "application/toml" || + mime === "image/svg+xml" || + mime.endsWith("+json") || + mime.endsWith("+xml") || + mime.endsWith("+yaml"); + return textual ? `${contentType}; charset=utf-8` : contentType; +} + export function pipeToResponse( res: ServerResponse, stream: NodeJS.ReadableStream & { destroy?: () => void }, diff --git a/src/api/routes/admin/files.ts b/src/api/routes/admin/files.ts index 49af4e542..9e69cc6ad 100644 --- a/src/api/routes/admin/files.ts +++ b/src/api/routes/admin/files.ts @@ -2,7 +2,7 @@ import { parseScopeId } from "../../../types.ts"; import { ByteSourceTooLargeError } from "../../../files/durable-byte-store.ts"; import { fileArtifactId } from "../../../files/file-artifact-store.ts"; import { MAX_ATTACHMENT_BYTES, mimeFromName, safeAttachmentName } from "../../../core/attachments.ts"; -import { contentDispositionAttachment, pipeToResponse, sendJson } from "../../http.ts"; +import { contentDispositionAttachment, contentTypeWithUtf8Charset, pipeToResponse, sendJson } from "../../http.ts"; import { audit, authorizeAdmin, requireScopedAdmin } from "../shared.ts"; import { type ApiCtx } from "../route.ts"; import { discoverScopes, FILES_PAGE_SIZE } from "./common.ts"; @@ -68,7 +68,7 @@ export async function downloadAdminFile(ctx: ApiCtx): Promise { const opened = await deps.files.open(id); if (!opened) return sendJson(res, 404, { error: "not_found" }); res.writeHead(200, { - "content-type": inline ? mime : "application/octet-stream", + "content-type": inline ? contentTypeWithUtf8Charset(art.mimetype) : "application/octet-stream", "content-length": String(opened.sizeBytes), "content-disposition": contentDispositionAttachment(art.name, inline ? "inline" : "attachment"), "x-content-type-options": "nosniff", diff --git a/src/api/routes/admin/scope-config.ts b/src/api/routes/admin/scope-config.ts index 22acc4f7e..17d206168 100644 --- a/src/api/routes/admin/scope-config.ts +++ b/src/api/routes/admin/scope-config.ts @@ -132,10 +132,25 @@ export async function listAdminScopes(ctx: ApiCtx): Promise { const crons = await app.listCrons(); const deployments = await app.listDeployments(); const skills = await app.listSkills(); + const environmentRows = await app.listEnvironments(); + const environments = environmentRows.map(({ environment, attachments }) => ({ + id: environment.id, + name: environment.name, + ownerActorId: environment.ownerActorId, + attachedScopes: attachments.map((attachment) => attachment.scopeId).sort(), + })); + const environmentById = new Map(environments.map((environment) => [environment.id, environment])); + const attachmentByScope = new Map( + environments.flatMap((environment) => + environment.attachedScopes.map((attachedScope) => [attachedScope, environment] as const), + ), + ); const owners = [ ...crons.map((c) => c.ownerScopeId), ...deployments.map((d) => d.ownerScopeId), ...skills.map((s) => s.scopeId), + ...environments.map((environment) => environment.id), + ...environments.flatMap((environment) => environment.attachedScopes), ]; const labels = await discoverScopes(app, deps, owners); const countBy = (ids: string[]): Map => { @@ -171,18 +186,31 @@ export async function listAdminScopes(ctx: ApiCtx): Promise { const cronN = countBy(crons.map((c) => c.ownerScopeId)); const deployN = countBy(deployments.map((d) => d.ownerScopeId)); const skillN = countBy(skills.map((s) => s.scopeId)); - const scopes = [...labels].map(([id, label]) => ({ - scopeId: id, - ...(label ? { label } : {}), - sessions: sessionN.get(id) ?? 0, - backgroundSessions: backgroundN.get(id) ?? 0, - lastActivity: lastActivityBy.get(id) ?? 0, - lastConversationActivity: lastConversationBy.get(id) ?? 0, - lastMessage: lastMessageBy.get(id) ?? "", - crons: cronN.get(id) ?? 0, - deployments: deployN.get(id) ?? 0, - skills: skillN.get(id) ?? 0, - })); + const scopes = [...labels].map(([id, label]) => { + const environment = environmentById.get(id); + const attachment = attachmentByScope.get(id); + return { + scopeId: id, + ...(label ? { label } : {}), + ...(environment?.name ? { environmentName: environment.name } : {}), + ...(attachment + ? { + environmentAttachment: { + environmentId: attachment.id, + environmentName: attachment.name, + }, + } + : {}), + sessions: sessionN.get(id) ?? 0, + backgroundSessions: backgroundN.get(id) ?? 0, + lastActivity: lastActivityBy.get(id) ?? 0, + lastConversationActivity: lastConversationBy.get(id) ?? 0, + lastMessage: lastMessageBy.get(id) ?? "", + crons: cronN.get(id) ?? 0, + deployments: deployN.get(id) ?? 0, + skills: skillN.get(id) ?? 0, + }; + }); scopes.sort( (a, b) => b.lastActivity - a.lastActivity || @@ -190,7 +218,35 @@ export async function listAdminScopes(ctx: ApiCtx): Promise { b.backgroundSessions - a.backgroundSessions || a.scopeId.localeCompare(b.scopeId), ); - return sendJson(res, 200, { scopeId: scope, scopes }); + return sendJson(res, 200, { scopeId: scope, scopes, environments }); +} + +interface ScopeEnvironmentMetadata { + environment?: { id: string; name: string; ownerActorId: string | null }; + environmentAttachment?: { environmentId: string; environmentName: string | null }; +} + +async function scopeEnvironmentMetadata(deps: ApiCtx["deps"], targetScope: string): Promise { + const store = deps.environments; + if (!store) return {}; + + const [environment, attachment] = await Promise.all([store.get(targetScope), store.getAttachment(targetScope)]); + const metadata: ScopeEnvironmentMetadata = {}; + if (environment?.name) { + metadata.environment = { + id: environment.id, + name: environment.name, + ownerActorId: environment.ownerActorId, + }; + } + if (attachment) { + const attachedEnvironment = await store.get(attachment.environmentId); + metadata.environmentAttachment = { + environmentId: attachment.environmentId, + environmentName: attachedEnvironment?.name ?? null, + }; + } + return metadata; } export async function getScopeConfig(ctx: ApiCtx): Promise { @@ -202,6 +258,7 @@ export async function getScopeConfig(ctx: ApiCtx): Promise { if (!actor) return; await deps.config.refreshScope(targetScope); audit(deps, { principalId: actor.id, action: "config.read", resource: "config", scopeLabel: targetScope }); + const environmentMetadata = await scopeEnvironmentMetadata(deps, targetScope); const serviceCredentials = await Promise.all( (deps.serviceCreds ? await deps.serviceCreds.listServiceCredentials(targetScope) : []).map(async (c) => { const usage = (await deps.credentialUsage?.list({ slug: c.slug, limit: 5000 })) ?? []; @@ -261,6 +318,7 @@ export async function getScopeConfig(ctx: ApiCtx): Promise { }; return sendJson(res, 200, { scopeId: targetScope, + ...environmentMetadata, ...values, soulVersion: deps.config.soulVersion(targetScope), soulHistory: deps.config.soulHistory(targetScope), diff --git a/src/api/routes/deployments.ts b/src/api/routes/deployments.ts index bff8821ef..53273a087 100644 --- a/src/api/routes/deployments.ts +++ b/src/api/routes/deployments.ts @@ -322,7 +322,8 @@ function proxyReachHttp2( return start(true); } if (res.destroyed || res.writableEnded) return; - if (!res.headersSent) sendJson(res, 502, { error: "bad_gateway", message: "deployment unreachable" }); + if (wantsWarmingPage(req, method) && !res.headersSent) sendWarmingPage(res); + else if (!res.headersSent) sendJson(res, 502, { error: "bad_gateway", message: "deployment unreachable" }); else res.destroy(); }; up.once("close", () => { @@ -330,14 +331,20 @@ function proxyReachHttp2( if (!responseStarted || !up.readableEnded || up.rstCode !== http2Constants.NGHTTP2_NO_ERROR) fail(); else if (!failureHandled && !res.destroyed && !res.writableEnded) res.end(); }); - up.setTimeout(deps.deployDialTimeoutMs ?? CONFIG_DEFAULTS.deployDialTimeoutMs, () => { - if (failureHandled) return; - failureHandled = true; - if (!res.headersSent) sendJson(res, 504, { error: "gateway_timeout", message: "deployment did not respond" }); - else res.end(); - up.close(http2Constants.NGHTTP2_CANCEL); - checkDeploymentHttp2Session(connection); - }); + const htmlNav = wantsWarmingPage(req, method); + up.setTimeout( + warmingDialTimeoutMs(`${host}:${port}`, htmlNav, deps.deployDialTimeoutMs ?? CONFIG_DEFAULTS.deployDialTimeoutMs), + () => { + if (failureHandled) return; + failureHandled = true; + if (htmlNav && !res.headersSent) sendWarmingPage(res); + else if (!res.headersSent) + sendJson(res, 504, { error: "gateway_timeout", message: "deployment did not respond" }); + else res.end(); + up.close(http2Constants.NGHTTP2_CANCEL); + checkDeploymentHttp2Session(connection); + }, + ); up.on("response", (responseHeaders) => { if (failureHandled || res.headersSent || res.destroyed || res.writableEnded) { up.close(http2Constants.NGHTTP2_CANCEL); @@ -345,6 +352,7 @@ function proxyReachHttp2( } responseStarted = true; up.setTimeout(0); + markUpstreamUp(`${host}:${port}`); armThrottleShield(`${host}:${port}`, Number(responseHeaders[":status"] ?? 0), up); const status = Number(responseHeaders[":status"] ?? 502); const safeHeaders = gatewaySafeResponseHeaders(responseHeaders); @@ -359,6 +367,80 @@ function proxyReachHttp2( start(false); } +// --- cold-start warming page ------------------------------------------------- +// AWS microVMs auto-resume on first connect, which can take many seconds. During +// that window a browser navigation would otherwise hang for the full dial timeout +// and then land on raw gateway JSON. For document requests we instead answer +// quickly with a small self-refreshing "warming up" page. +const WARM_RECENT_MS = 60_000; +const COLD_FIRST_BYTE_TIMEOUT_MS = 4_000; +const upstreamLastOk = new Map(); + +function markUpstreamUp(upstreamKey: string): void { + if (upstreamLastOk.size > 1000) { + for (const [k, at] of upstreamLastOk) if (Date.now() - at > WARM_RECENT_MS) upstreamLastOk.delete(k); + } + upstreamLastOk.set(upstreamKey, Date.now()); +} + +function wantsWarmingPage(req: BaseCtx["req"], method: string): boolean { + if (method !== "GET" && method !== "HEAD") return false; + const dest = String(req.headers["sec-fetch-dest"] ?? ""); + if (dest && dest !== "document") return false; + return String(req.headers.accept ?? "").includes("text/html"); +} + +function warmingDialTimeoutMs(upstreamKey: string, htmlNav: boolean, configuredMs: number): number { + if (!htmlNav) return configuredMs; + const lastOk = upstreamLastOk.get(upstreamKey) ?? 0; + if (Date.now() - lastOk < WARM_RECENT_MS) return configuredMs; + return Math.min(configuredMs, COLD_FIRST_BYTE_TIMEOUT_MS); +} + +const WARMING_PAGE_HTML = ` + +Starting up… + +

Starting up…

+

+`; + +function sendWarmingPage(res: BaseCtx["res"]): void { + if (res.headersSent || res.destroyed || res.writableEnded) { + res.end(); + return; + } + res.writeHead(503, { + "content-type": "text/html; charset=utf-8", + "cache-control": "no-store", + "retry-after": "2", + }); + res.end(WARMING_PAGE_HTML); +} +// ----------------------------------------------------------------------------- + async function proxyReach( ctx: BaseCtx, reach: Awaited>, @@ -411,21 +493,30 @@ async function proxyReach( proxyReachHttp2(ctx, reach, subPath, headers, bufferedBody); return; } + const htmlNav = wantsWarmingPage(req, method); const up = requestFn({ hostname: host, port, path: subPath + url.search, method, headers }, (upRes) => { up.setTimeout(0); + markUpstreamUp(upstreamKey); upRes.on("error", () => res.destroy()); armThrottleShield(upstreamKey, upRes.statusCode ?? 0, upRes); const headers = gatewaySafeResponseHeaders(upRes.headers); res.writeHead(upRes.statusCode ?? 502, headers); upRes.pipe(res); }); - up.setTimeout(deps.deployDialTimeoutMs ?? CONFIG_DEFAULTS.deployDialTimeoutMs, () => { - if (!res.headersSent) sendJson(res, 504, { error: "gateway_timeout", message: "deployment did not respond" }); + const dialMs = warmingDialTimeoutMs( + upstreamKey, + htmlNav, + deps.deployDialTimeoutMs ?? CONFIG_DEFAULTS.deployDialTimeoutMs, + ); + up.setTimeout(dialMs, () => { + if (htmlNav && !res.headersSent) sendWarmingPage(res); + else if (!res.headersSent) sendJson(res, 504, { error: "gateway_timeout", message: "deployment did not respond" }); else res.end(); up.destroy(); }); up.on("error", () => { - if (!res.headersSent) sendJson(res, 502, { error: "bad_gateway", message: "deployment unreachable" }); + if (htmlNav && !res.headersSent) sendWarmingPage(res); + else if (!res.headersSent) sendJson(res, 502, { error: "bad_gateway", message: "deployment unreachable" }); else res.end(); }); req.on("error", () => up.destroy()); diff --git a/src/api/routes/surface.ts b/src/api/routes/surface.ts index 3fbe270af..6c0b73e35 100644 --- a/src/api/routes/surface.ts +++ b/src/api/routes/surface.ts @@ -18,7 +18,7 @@ import { builtInModelCatalog, selectableCatalogForHarness, selectableModelCatalo import { errMessage } from "../../util/errors.ts"; import { renderAgentApis } from "../agent-api-catalog.ts"; import { mintCapabilityToken, CAPABILITY_TTL_MS } from "../../auth/capability-token.ts"; -import { pipeToResponse, sendJson } from "../http.ts"; +import { contentTypeWithUtf8Charset, pipeToResponse, sendJson } from "../http.ts"; import { audit, isObj, orgScope } from "./shared.ts"; import { type ApiCtx, type Route } from "./route.ts"; import { @@ -103,18 +103,27 @@ async function spawnAgentConversation(ctx: ApiCtx): Promise { }); if (!out) return sendJson(res, 404, { error: "not_found", message: "cannot start a session in this scope" }); const session = out.session; + const sessionScope = parseScopeId(session.scopeId); const turn = await app.turn({ surface: session.surface ?? "web", actor: { externalId: capability.actorId }, conversation: { kind: session.type, threadRef: session.threadRef, + ...(sessionScope.kind === "channel" || sessionScope.kind === "group" ? { channelRef: sessionScope.ref } : {}), ...(session.channelName ? { channelName: session.channelName } : {}), }, text: b.text, spawned: true, async: true, }); + if (turn.status === "refused") { + await app.discardSession(session.id, capability.actorId); + return sendJson(res, 409, { + error: "seed_turn_refused", + message: (turn as { reason?: string }).reason ?? "the first message was refused", + }); + } const runId = (turn as { runId?: string }).runId; return sendJson(res, 202, { session, turn: { status: turn.status, ...(runId ? { runId } : {}) } }); } @@ -252,7 +261,7 @@ async function getFileContent(ctx: ApiCtx): Promise { const opened = await app.openFileForViewer(id, viewer); if (!opened) return sendJson(res, 404, { error: "not_found" }); res.writeHead(200, { - "content-type": opened.mimetype || "application/octet-stream", + "content-type": contentTypeWithUtf8Charset(opened.mimetype || "application/octet-stream"), "content-length": String(opened.sizeBytes), "content-disposition": `inline; filename*=UTF-8''${encodeURIComponent(opened.name)}`, }); diff --git a/src/connectors/background-exec-broker.ts b/src/connectors/background-exec-broker.ts index b01a2fd0f..ce7c44f8c 100644 --- a/src/connectors/background-exec-broker.ts +++ b/src/connectors/background-exec-broker.ts @@ -1,7 +1,7 @@ import type { ProcessSandbox, ProcessState, SandboxHandle } from "../sandbox/sandbox.ts"; import type { ProcessRegistry, ProcessStatus } from "../processes/process-registry.ts"; import { awaitProcessExit } from "../sandbox/await-process-exit.ts"; -import { pollProcess } from "../sandbox/process-poll.ts"; +import { pollProcess, processIsGone } from "../sandbox/process-poll.ts"; import { redactCommand } from "../sandbox/exec-process-session.ts"; import { CONFIG_DEFAULTS } from "../config.ts"; @@ -108,7 +108,8 @@ export function createBackgroundBroker(deps: BackgroundExecBrokerDeps): Backgrou await deps.registry.markStatus(existingProcessId, "exited"); processId = null; } - } catch { + } catch (error) { + if (!processIsGone(error)) throw error; await deps.registry.delete(existingProcessId); processId = null; } diff --git a/src/core/orchestrator.ts b/src/core/orchestrator.ts index cadd8abae..fdb800127 100644 --- a/src/core/orchestrator.ts +++ b/src/core/orchestrator.ts @@ -25,7 +25,7 @@ import type { GapPhase, LeaseAttempt, SessionStore } from "../sessions/session-s import { isOverheardEntry } from "../sessions/session-store.ts"; import { supportsProcessSessions, supportsScopeProfile } from "../sandbox/sandbox.ts"; import { createBackgroundBroker } from "../connectors/background-exec-broker.ts"; -import { createMonitorBroker } from "../monitors/monitor-broker.ts"; +import { createMonitorBroker, readBackgroundOutputTail } from "../monitors/monitor-broker.ts"; import { isPollSurface, isSilentPollReply } from "../triggers/run-trigger.ts"; import { envKey } from "../credentials/connector-token.ts"; import { renderKeychainManifest, type MaterializedEnvCred } from "../credentials/keychain.ts"; @@ -71,7 +71,9 @@ import { renderPendingOnboardingPrompt, } from "../onboarding/onboarding.ts"; import { createToolContext, NeedsApproval, CommandDenied } from "../tools/primitives.ts"; -import type { BrokeredLayerTool } from "../deployment/load-layer.ts"; +import { evaluateCommandWithLayer } from "../policy/command-policy.ts"; +import { createSecretValueMasker } from "../security/secret-masking.ts"; +import { shq } from "../util/shell.ts"; import type { FileArtifact } from "../files/file-artifact-store.ts"; import { filterHistoryForAudience, principalEntitledToScope } from "../resolution/context-filter.ts"; import { @@ -922,6 +924,15 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { commandUses.set(key, n - 1); return true; }; + const authorizeCommand = (command: string, approvalKey?: string): boolean => { + let key = approvalKey ?? command; + if (approvalKey !== undefined && commandUses.has(approvalKey)) key = approvalKey; + else if (commandUses.has(command)) key = command; + const n = commandUses.get(key) ?? 0; + if (n <= 0) return false; + commandUses.set(key, n - 1); + return true; + }; const brokeredTools = deps.brokeredTools ?? []; const cutoverModes = new Map(); for (const tool of brokeredTools) { @@ -943,9 +954,15 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { input.origin.kind === "automation" && input.origin.useOwnerKeychain === true; let ownerAuthAvailable = isolateOwnerKeychain; + if ( + deps.sharedOwnerAuthIsolation === true && + conversation.kind !== "dm" && + brokeredTools.some((tool) => cutoverModeOf(tool.service) !== "legacy" && deps.layerBrokerFor?.(tool)) + ) { + ownerAuthAvailable = true; + } const connectorEnv: Record = {}; const ownerAuthEnv: Record = {}; - const brokerVended = new Map }>(); const ownerEnvCredentialIds: string[] = []; const keychainInjected: MaterializedEnvCred[] = []; const credsStart = Date.now(); @@ -1159,17 +1176,9 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { if (!strictReadOnly && actor.type === "internal") { for (const tool of brokeredTools) { const mode = cutoverModeOf(tool.service); + if (mode !== "legacy") continue; const broker = deps.layerBrokerFor?.(tool); if (!broker) { - if (conversation.kind !== "dm" && mode !== "legacy") { - deps.credentialUsage?.record({ - slug: tool.service, - host: "sts.amazonaws.com", - status: mode === "ephemeral_only" ? "ephemeral_failed_closed" : "legacy_fallback", - scopeLabel: scopeId, - principalId: actor.id, - }); - } continue; } const aws = await broker @@ -1184,19 +1193,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { AWS_DEFAULT_REGION: aws.region, } : null; - const isolateShared = - deps.sharedOwnerAuthIsolation === true && conversation.kind !== "dm" && mode !== "legacy"; - if (awsEnv && isolateShared) { - brokerVended.set(tool.service, { tool, env: awsEnv }); - ownerAuthAvailable = true; - deps.credentialUsage?.record({ - slug: tool.service, - host: "sts.amazonaws.com", - status: "ephemeral_vended", - scopeLabel: scopeId, - principalId: actor.id, - }); - } else if (awsEnv && (conversation.kind === "dm" || mode === "legacy")) { + if (awsEnv) { Object.assign(connectorEnv, awsEnv); deps.credentialUsage?.record({ slug: tool.service, @@ -1206,13 +1203,10 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { principalId: actor.id, }); } else { - let status = "legacy_unavailable"; - if (mode === "ephemeral_only") status = "ephemeral_failed_closed"; - else if (mode === "prefer_ephemeral") status = "legacy_fallback"; deps.credentialUsage?.record({ slug: tool.service, host: "sts.amazonaws.com", - status, + status: "legacy_unavailable", scopeLabel: scopeId, principalId: actor.id, }); @@ -1230,7 +1224,16 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { swallow("gap-work emit", e); } }; - const commandPolicy = resolution.commandPolicy; + const ephemeralOnlyDenyRules = brokeredTools + .filter((candidate) => cutoverModeOf(candidate.service) === "ephemeral_only") + .map((tool) => ({ + pattern: `(^|[\\s;&|()])${tool.binary.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}($|[\\s;&|()])`, + decision: "deny" as const, + reason: `credential-bearing service ${tool.service} must be run with credential_exec`, + })); + const commandPolicy = ephemeralOnlyDenyRules.length + ? { ...resolution.commandPolicy, rules: [...ephemeralOnlyDenyRules, ...resolution.commandPolicy.rules] } + : resolution.commandPolicy; const layerCommandRules = [...(deps.deploymentLayer?.commandRules ?? [])]; const reachAvailable = !!deps.reachExec && !!deps.directory && conversation.kind === "dm"; const { @@ -1238,6 +1241,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { scratchBox, ownerAuthBox, ownerAuthCommand, + scopedCommand, provision, provisionScratch, provisionOwnerAuth, @@ -1263,7 +1267,6 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { ownerAuthAvailable, ownerAuthEnv, ownerEnvCredentialIds, - brokerVended, brokeredTools, quarantinedServices, brokerCutoverServices, @@ -1569,11 +1572,30 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { }) : undefined; + const readOutputTail = backgroundBroker + ? async (processId: string, maxBytes: number) => { + const handle = await provision(); + return readBackgroundOutputTail(maxBytes, async (cursor, readMaxBytes) => { + const read = await backgroundBroker.poll(handle, processId, { + sinceCursor: cursor, + maxBytes: readMaxBytes, + waitMs: 0, + }); + return { + chunks: read.chunks, + cursor: read.cursor, + ...(read.status.state === "exited" ? { exitCode: read.status.code } : {}), + }; + }); + } + : undefined; + const monitorBroker = deps.monitors && deps.processes && supportsProcessSessions(deps.sandbox) ? createMonitorBroker({ store: deps.monitors, registry: deps.processes, + readOutputTail: readOutputTail ?? (async () => ({ outputTail: "" })), scopeId: memoryScopeId, owner: actor.id, ownerScopeId: scopeId, @@ -1671,6 +1693,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { provisionScratch, ...(provisionOwnerAuth ? { provisionOwnerAuth } : {}), ...(ownerAuthCommand ? { ownerAuthCommand } : {}), + ...(scopedCommand ? { scopedCommand } : {}), ensureSkillTree, ...(reachAvailable ? { @@ -1684,15 +1707,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { layers: resolution.layers, commandPolicy: () => commandPolicy, layerCommandRules: () => layerCommandRules, - authorizeCommand: (command: string, approvalKey?: string) => { - let key = approvalKey ?? command; - if (approvalKey !== undefined && commandUses.has(approvalKey)) key = approvalKey; - else if (commandUses.has(command)) key = command; - const n = commandUses.get(key) ?? 0; - if (n <= 0) return false; - commandUses.set(key, n - 1); - return true; - }, + authorizeCommand, grantedHandles: resolution.grantedHandles, sharedMaterializeDir: turnSharedDir, workspace: deps.workspace, @@ -1701,6 +1716,132 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { files: deps.files, auditLog: deps.auditLog, createdBy: actor.id, + ...(() => { + const available = + strictReadOnly || actor.type !== "internal" + ? [] + : brokeredTools.filter( + (tool) => cutoverModeOf(tool.service) !== "legacy" && deps.layerBrokerFor?.(tool), + ); + if (!available.length) return {}; + return { + credentialExecServices: available.map(({ service, binary }) => ({ service, binary })), + credentialExec: async ( + service: string, + args: string[], + opts?: { timeoutSeconds?: number; signal?: AbortSignal }, + ) => { + const tool = available.find((candidate) => candidate.service === service); + if (!tool || cutoverModeOf(service) === "legacy") { + throw new Error(`credential_exec service is unavailable: ${service}`); + } + const broker = deps.layerBrokerFor?.(tool); + if (!broker) throw new Error(`credential_exec broker is unavailable: ${service}`); + const composed = [shq(tool.binary), ...args.map(shq)].join(" "); + const gate = evaluateCommandWithLayer( + composed, + resolution.commandPolicy, + deps.deploymentLayer?.commandRules ?? [], + ); + if (gate.decision === "deny") throw new CommandDenied(composed, gate.reason ?? "denied by policy"); + if (gate.decision === "require_approval" && !authorizeCommand(composed, gate.approvalKey)) { + throw new NeedsApproval( + composed, + gate.reason ?? "requires approval", + "approval", + gate.matched, + gate.approvalKey, + ); + } + let aws; + try { + aws = await broker.credsForActor(actor.id); + } catch { + deps.credentialUsage?.record({ + slug: service, + host: "sts.amazonaws.com", + status: cutoverModeOf(service) === "ephemeral_only" ? "ephemeral_failed_closed" : "legacy_fallback", + scopeLabel: scopeId, + principalId: actor.id, + }); + throw new Error(`credential_exec could not vend credentials for ${service}`); + } + const awsEnv = { + AWS_ACCESS_KEY_ID: aws.accessKeyId, + AWS_SECRET_ACCESS_KEY: aws.secretAccessKey, + AWS_SESSION_TOKEN: aws.sessionToken, + AWS_REGION: aws.region, + AWS_DEFAULT_REGION: aws.region, + }; + const mask = createSecretValueMasker(awsEnv); + let handle; + let result: Awaited> | undefined; + let runError: unknown; + let cleanupError: unknown; + try { + handle = await deps.sandbox.provision( + resolution.layers.filter((layer) => layer.mode === "ro" && layer.mountPath === "global"), + { + env: awsEnv, + egress: resolution.egress, + ...(egressTokenForTurn ? { egressToken: egressTokenForTurn } : {}), + scratch: { key: `credential-exec:${session.id}:${randomUUID()}` }, + routeScopeId: memoryScopeId, + }, + ); + deps.credentialUsage?.record({ + slug: service, + host: "sts.amazonaws.com", + status: "ephemeral_vended", + scopeLabel: scopeId, + principalId: actor.id, + }); + deps.auditLog.record({ + at: Date.now(), + principalId: actor.id, + action: "credential.materialize", + resource: `${service} (ephemeral broker)`, + scopeLabel: scopeId, + }); + const requestedMs = opts?.timeoutSeconds == null ? deps.execTimeoutMs : opts.timeoutSeconds * 1000; + const timeoutMs = + requestedMs != null && deps.execTimeoutCeilingMs != null + ? Math.min(requestedMs, deps.execTimeoutCeilingMs) + : requestedMs; + result = await deps.sandbox.run( + handle, + composed, + timeoutMs !== undefined || opts?.signal + ? { + ...(timeoutMs !== undefined ? { timeoutMs } : {}), + ...(opts?.signal ? { signal: opts.signal } : {}), + } + : undefined, + ); + } catch (error) { + runError = error; + } finally { + if (handle) { + let lastError: unknown; + for (let attempt = 1; attempt <= 3; attempt++) { + try { + await deps.sandbox.teardown(handle, { destroy: true }); + lastError = undefined; + break; + } catch (error) { + lastError = error; + if (attempt < 3) await sleep(50 * attempt); + } + } + cleanupError = lastError; + } + } + if (cleanupError) throw new Error(`credential_exec cleanup failed for ${service}`); + if (runError || !result) throw new Error(`credential_exec failed while running ${service}`); + return { ...result, stdout: mask(result.stdout), stderr: mask(result.stderr) }; + }, + }; + })(), ...(deps.publicWebUrl ? { publicWebUrl: deps.publicWebUrl } : {}), publishContext: { conversationKind: conversation.kind, @@ -2150,6 +2291,7 @@ export function createOrchestrator(deps: OrchestratorDeps): Orchestrator { systemCacheBoundary: stableSystemBytes, history: continuation?.history ?? history, tools, + ...(tools.credentialExecServices ? { credentialExecServices: tools.credentialExecServices } : {}), ...(securityPolicy.inboundScreening === "external" && (deps.securityScreener || deps.harness.models.screenSecurity) ? { diff --git a/src/core/orchestrator/sandboxes.ts b/src/core/orchestrator/sandboxes.ts index 67591391d..6592de451 100644 --- a/src/core/orchestrator/sandboxes.ts +++ b/src/core/orchestrator/sandboxes.ts @@ -14,7 +14,6 @@ import { type ResidentAuthConnector, } from "../../credentials/resident-auth.ts"; import { shq } from "../../util/shell.ts"; -import type { BrokeredLayerTool } from "../../deployment/load-layer.ts"; import { createSkillMaterializer, safeSkillDirName } from "../../skills/materialize.ts"; import type { SkillResolution } from "../../skills/skill-store.ts"; import { TURN_FILES_DIR } from "../attachments.ts"; @@ -43,8 +42,7 @@ export interface TurnSandboxContext { ownerAuthAvailable: boolean; ownerAuthEnv: Record; ownerEnvCredentialIds: string[]; - brokerVended: Map }>; - brokeredTools: readonly BrokeredLayerTool[]; + brokeredTools: readonly import("../../deployment/load-layer.ts").BrokeredLayerTool[]; quarantinedServices: string[]; brokerCutoverServices: string[]; cutoverModeOf: (service: string) => DeviceFlowCutoverMode; @@ -76,7 +74,6 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) { ownerAuthAvailable, ownerAuthEnv, ownerEnvCredentialIds, - brokerVended, brokeredTools, quarantinedServices, brokerCutoverServices, @@ -91,6 +88,20 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) { } = ctx; let ownerAuthCommand: ((command: string) => string) | undefined; + const brokerEnvKeys = [ + "AWS_ACCESS_KEY_ID", + "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", + "AWS_REGION", + "AWS_DEFAULT_REGION", + ]; + const unsetBrokerEnv = (env: Record): string => { + const keys = brokerEnvKeys.filter((key) => !(key in env)); + return keys.length ? `unset ${keys.join(" ")}; ` : ""; + }; + const scopedCommand = brokerCutoverServices.length + ? (command: string): string => `${unsetBrokerEnv(connectorEnv)}${command}` + : undefined; if (ownerAuthAvailable) { ownerAuthCommand = (command) => { for (const credentialId of ownerEnvCredentialIds) { @@ -102,30 +113,10 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) { scopeLabel: scopeId, }); } - const invoked = [...brokerVended.values()].filter(({ tool }) => - new RegExp(`(^|[\\s;&|()])${tool.binary}(?=$|[\\s;&|()])`).test(command), - ); - for (const { tool } of invoked) { - deps.auditLog.record({ - at: Date.now(), - principalId: actor.id, - action: "credential.materialize", - resource: `${tool.service} (ephemeral broker)`, - scopeLabel: scopeId, - }); - } const exports = Object.entries(ownerAuthEnv) .map(([key, value]) => `${key}=${shq(value)}`) .join(" "); - const wrappers = invoked - .map(({ tool, env }) => { - const brokerExports = Object.entries(env) - .map(([key, value]) => `${key}=${shq(value)}`) - .join(" "); - return `${tool.binary}() { ${brokerExports} command ${tool.binary} "$@"; }; `; - }) - .join(""); - return `${exports ? `export ${exports}; ` : ""}${wrappers}${command}`; + return `unset AGENT_API_TOKEN AGENT_OAUTH_CONSENT_TOKEN AGENT_CREDENTIAL_TOKEN AGENT_OUTBOX; ${unsetBrokerEnv(ownerAuthEnv)}${exports ? `export ${exports}; ` : ""}${command}`; }; } const box: { @@ -583,6 +574,7 @@ export function createTurnSandboxes(ctx: TurnSandboxContext) { scratchBox, ownerAuthBox, ownerAuthCommand, + scopedCommand, provision, provisionScratch, provisionOwnerAuth, diff --git a/src/core/orchestrator/surface-tools.ts b/src/core/orchestrator/surface-tools.ts index 42ced8cc9..2634b4954 100644 --- a/src/core/orchestrator/surface-tools.ts +++ b/src/core/orchestrator/surface-tools.ts @@ -417,9 +417,14 @@ export function createSurfaceToolDeps(ctx: SurfaceToolsContext): SurfaceToolDeps if (conversation.kind === "dm" || !conversation.channelRef) return { ok: false, message: "standing orders are per-channel — there isn't one for a DM." }; const p = await deps.channelPolicy.get(conversation.channelRef); - return { ok: true, orders: p?.orders ?? "", ...(p?.bots && Object.keys(p.bots).length ? { bots: p.bots } : {}) }; + return { + ok: true, + orders: p?.orders ?? "", + ...(p?.bots && Object.keys(p.bots).length ? { bots: p.bots } : {}), + ...(p?.ambientEnabled !== undefined ? { ambientEnabled: p.ambientEnabled } : {}), + }; }, - setStandingOrder: async (orders: string, bots?: Record) => { + setStandingOrder: async (orders: string, bots?: Record, ambientEnabled?: boolean | null) => { if (!deps.channelPolicy) return { ok: false, message: "standing orders aren't available on this turn" }; if (conversation.kind === "dm" || !conversation.channelRef) return { ok: false, message: "standing orders are per-channel — you can only set one from inside a channel." }; @@ -433,6 +438,7 @@ export function createSurfaceToolDeps(ctx: SurfaceToolsContext): SurfaceToolDeps setBy: actor.id, bots: parsedBots, sessionId: session.id, + ambientEnabled, }); deps.auditLog.record({ at: Date.now(), @@ -441,7 +447,12 @@ export function createSurfaceToolDeps(ctx: SurfaceToolsContext): SurfaceToolDeps resource: conversation.channelRef, scopeLabel: scopeId, }); - return { ok: true, orders, ...(p.bots && Object.keys(p.bots).length ? { bots: p.bots } : {}) }; + return { + ok: true, + orders, + ...(p.bots && Object.keys(p.bots).length ? { bots: p.bots } : {}), + ...(p.ambientEnabled !== undefined ? { ambientEnabled: p.ambientEnabled } : {}), + }; }, staySilent: async (reason: string) => { spine.staySilentReason = reason; diff --git a/src/cron/cron-store.ts b/src/cron/cron-store.ts index c50c00a47..8a75502df 100644 --- a/src/cron/cron-store.ts +++ b/src/cron/cron-store.ts @@ -43,6 +43,7 @@ export interface CronStore { setRecipientConsent(id: string, recipientConsent: RecipientConsent): Promise; recordFire(id: string, entry: CronFireLogEntry): Promise; markFired(id: string, at: number, scheduledAt?: number): Promise; + markAttempted(id: string, at: number): Promise; claimSlot(id: string, scheduledAt: number, at: number): Promise; unclaimSlot(id: string, scheduledAt: number, at: number, priorLastFiredAt: number | undefined): Promise; due(now: number): Promise>; @@ -175,6 +176,9 @@ export function createCronStore(backing: DurableMap = createMemoryMap = []; for (const c of await backing.all()) { diff --git a/src/cron/scheduler.ts b/src/cron/scheduler.ts index 066d79114..333bbbc4a 100644 --- a/src/cron/scheduler.ts +++ b/src/cron/scheduler.ts @@ -174,13 +174,28 @@ export function createScheduler(deps: SchedulerDeps): Scheduler { const fireDue = async (t: number): Promise => { const due = await deps.crons.due(t); - const batch = due.slice(0, maxFiresPerTick); - if (due.length > batch.length) { + let batch = due; + if (due.length > maxFiresPerTick) { + const ordered = [...due].sort((a, b) => (a.lastAttemptAt ?? 0) - (b.lastAttemptAt ?? 0)); + batch = []; + for (const cron of ordered) { + if (batch.length >= maxFiresPerTick) break; + try { + await deps.crons.markAttempted(cron.id, t); + batch.push(cron); + } catch (e) { + console.error("[scheduler] attempt mark failed, holding this cron back:", errMessage(e)); + } + } console.warn(`[scheduler] fan-out capped: firing ${batch.length}/${due.length} due crons this tick`); } for (const cron of batch) { - const { authzFailed } = await fire(cron, t, `cron:${cron.id}:${cron.scheduledAt}`, cron.scheduledAt); - if (!authzFailed) await deps.crons.markFired(cron.id, t, cron.scheduledAt); + try { + const { authzFailed } = await fire(cron, t, `cron:${cron.id}:${cron.scheduledAt}`, cron.scheduledAt); + if (!authzFailed) await deps.crons.markFired(cron.id, t, cron.scheduledAt); + } catch (e) { + console.error("[scheduler] fire failed:", errMessage(e)); + } } }; diff --git a/src/deployment/deployment-layer-store.ts b/src/deployment/deployment-layer-store.ts index c908bb02a..61c1d6bbd 100644 --- a/src/deployment/deployment-layer-store.ts +++ b/src/deployment/deployment-layer-store.ts @@ -232,6 +232,7 @@ function validateBundle( export function createDeploymentLayerStore(opts: { backing: DurableMap; runtime: DeploymentLayerRuntime; + validateBundle?: (bundle: DeploymentLayerBundle) => void; skills: SkillStore; skillBundles?: SkillBundleStore; scopeId: ScopeId; @@ -250,6 +251,11 @@ export function createDeploymentLayerStore(opts: { const queue = createKeyedQueue(); const advisoryLock = opts.advisoryLock ?? createNoopAdvisoryLock(); const withFleetLock = (fn: () => Promise): Promise => advisoryLock.withLock(SKILL_MATERIALIZATION_LOCK, fn); + const validatedBundle = (input: DeploymentLayerBundle, dir: string): ReturnType => { + const validated = validateBundle(input, dir); + opts.validateBundle?.(validated.bundle); + return validated; + }; const retrying = async (fn: () => Promise): Promise => { for (let attempt = 0; ; attempt++) { @@ -310,7 +316,7 @@ export function createDeploymentLayerStore(opts: { validated?: ReturnType, ): Promise => { if (opts.runtime.dir !== `durable:${record.contentHash}`) return false; - const next = validated ?? validateBundle(record.bundle, `durable:${record.contentHash}`); + const next = validated ?? validatedBundle(record.bundle, `durable:${record.contentHash}`); if (JSON.stringify(publicResolved(opts.runtime)) !== JSON.stringify(publicResolved(next.runtime))) return false; const current = layerSkills(await opts.skills.list()); const wanted = new Set(next.manifests.map((manifest) => manifest.name)); @@ -330,7 +336,7 @@ export function createDeploymentLayerStore(opts: { }; const apply = async (record: StoredDeploymentLayer): Promise => { - const { manifests, runtime: nextRuntime } = validateBundle(record.bundle, `durable:${record.contentHash}`); + const { manifests, runtime: nextRuntime } = validatedBundle(record.bundle, `durable:${record.contentHash}`); if ( appliedHash === record.contentHash && (await projectionMatches(record, { bundle: record.bundle, manifests, runtime: nextRuntime })) @@ -502,7 +508,7 @@ export function createDeploymentLayerStore(opts: { let manifests: SkillManifest[]; let runtime: DeploymentLayerRuntime; try { - const validated = validateBundle(input, "durable:pending"); + const validated = validatedBundle(input, "durable:pending"); bundle = validated.bundle; manifests = validated.manifests; runtime = validated.runtime; diff --git a/src/egress-authz-main.ts b/src/egress-authz-main.ts index c5cbe5c91..e484e80c9 100644 --- a/src/egress-authz-main.ts +++ b/src/egress-authz-main.ts @@ -18,12 +18,16 @@ const DENY_ALL: EgressPolicy = { allowedHosts: ["deny.invalid"], deniedHosts: [] const METADATA_HOSTS = ["metadata.google.internal", "metadata.goog"]; -const LINK_LOCAL = new BlockList(); -LINK_LOCAL.addSubnet("169.254.0.0", 16, "ipv4"); -LINK_LOCAL.addSubnet("fe80::", 10, "ipv6"); -LINK_LOCAL.addAddress("fd00:ec2::254", "ipv6"); - -export function isLinkLocalOrMetadataIp(ip: string): boolean { +const BLOCKED = new BlockList(); +BLOCKED.addSubnet("169.254.0.0", 16, "ipv4"); +BLOCKED.addSubnet("fe80::", 10, "ipv6"); +BLOCKED.addAddress("fd00:ec2::254", "ipv6"); +BLOCKED.addSubnet("127.0.0.0", 8, "ipv4"); +BLOCKED.addSubnet("0.0.0.0", 8, "ipv4"); +BLOCKED.addAddress("::1", "ipv6"); +BLOCKED.addAddress("::", "ipv6"); + +export function isBlockedDestinationIp(ip: string): boolean { const s = ip .trim() .toLowerCase() @@ -31,7 +35,7 @@ export function isLinkLocalOrMetadataIp(ip: string): boolean { .replace(/%.*$/, ""); const fam = isIP(s); if (!fam) return false; - return LINK_LOCAL.check(s, fam === 4 ? "ipv4" : "ipv6"); + return BLOCKED.check(s, fam === 4 ? "ipv4" : "ipv6"); } function isAlwaysBlockedHost(host: string): boolean { @@ -40,7 +44,7 @@ function isAlwaysBlockedHost(host: string): boolean { .toLowerCase() .replace(/^\[(.*)\]$/, "$1") .replace(/\.$/, ""); - if (isIP(h)) return isLinkLocalOrMetadataIp(h); + if (isIP(h)) return isBlockedDestinationIp(h); return METADATA_HOSTS.some((m) => h === m || h.endsWith(`.${m}`)); } @@ -108,7 +112,7 @@ async function decide( if ( ips.some( (ip) => - isLinkLocalOrMetadataIp(ip) || + isBlockedDestinationIp(ip) || isHostDenied(ip, policy?.deniedHosts) || (policy?.denyPrivateNetworks === true && !privateAllowed && isPrivateNetworkIp(ip)), ) diff --git a/src/files/durable-byte-store.ts b/src/files/durable-byte-store.ts index 7e2a86277..029a0f1a3 100644 --- a/src/files/durable-byte-store.ts +++ b/src/files/durable-byte-store.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { createReadStream, createWriteStream } from "node:fs"; import { mkdir, rename, rm, stat } from "node:fs/promises"; import { once } from "node:events"; @@ -84,7 +85,7 @@ export function createLocalDurableByteStore(dir: string): DurableByteStore { } catch (error) { void error; } - const partPath = join(base, `${sha256}.${process.pid}.part`); + const partPath = join(base, `${sha256}.${randomUUID()}.part`); const out = createWriteStream(partPath); try { if (!out.write(data)) await once(out, "drain"); @@ -95,7 +96,12 @@ export function createLocalDurableByteStore(dir: string): DurableByteStore { await rm(partPath, { force: true }).catch(swallowAs("files: partial-file cleanup", undefined)); throw err; } - await rename(partPath, finalPath); + try { + await rename(partPath, finalPath); + } catch (err) { + await rm(partPath, { force: true }).catch(swallowAs("files: partial-file cleanup", undefined)); + throw err; + } return { blobKey, sizeBytes: data.length, sha256 }; }, diff --git a/src/harness/claude-harness.ts b/src/harness/claude-harness.ts index 0d4bf6517..e07f15db3 100644 --- a/src/harness/claude-harness.ts +++ b/src/harness/claude-harness.ts @@ -199,7 +199,12 @@ function toolOptions(opts: ClaudeHarnessOptions, turn?: HarnessTurnInput): PiToo backgroundJobTtlMs: opts.backgroundJobTtlMs, backgroundJobTtlMaxMs: opts.backgroundJobTtlMaxMs, ...(turn - ? { readOnly: turn.readOnly, surfaceTools: turn.surfaceTools, surfaceName: turn.surfaceName } + ? { + readOnly: turn.readOnly, + surfaceTools: turn.surfaceTools, + surfaceName: turn.surfaceName, + credentialExecServices: turn.credentialExecServices, + } : { surfaceTools: true, surfaceName: "slack" }), }; } diff --git a/src/harness/codex-harness.ts b/src/harness/codex-harness.ts index e40eaf8f1..cfcb42f2f 100644 --- a/src/harness/codex-harness.ts +++ b/src/harness/codex-harness.ts @@ -232,7 +232,12 @@ function toolOptions(opts: CodexHarnessOptions, turn?: HarnessTurnInput): PiTool backgroundJobTtlMs: opts.backgroundJobTtlMs, backgroundJobTtlMaxMs: opts.backgroundJobTtlMaxMs, ...(turn - ? { readOnly: turn.readOnly, surfaceTools: turn.surfaceTools, surfaceName: turn.surfaceName } + ? { + readOnly: turn.readOnly, + surfaceTools: turn.surfaceTools, + surfaceName: turn.surfaceName, + credentialExecServices: turn.credentialExecServices, + } : { surfaceTools: true, surfaceName: "slack" }), }; } diff --git a/src/harness/harness.ts b/src/harness/harness.ts index 809909c73..970642a81 100644 --- a/src/harness/harness.ts +++ b/src/harness/harness.ts @@ -66,6 +66,7 @@ export interface HarnessTurnInput { systemCacheBoundary?: number; history: SessionEntry[]; tools: ToolContext; + credentialExecServices?: readonly { service: string; binary: string }[]; screenExternalContent?(input: { content: string; tool: string; diff --git a/src/harness/mock-harness.ts b/src/harness/mock-harness.ts index d9bbc39f8..73718230a 100644 --- a/src/harness/mock-harness.ts +++ b/src/harness/mock-harness.ts @@ -19,6 +19,7 @@ const READ_ONLY_BLOCKED_PREFIXES = [ "!run ", "!scratch ", "!owner ", + "!credential ", "!reach ", "!paused-approval ", "!collect-approval ", @@ -262,6 +263,22 @@ export function createMockHarness(): Harness { scopeLabel: turn.scopeLabel, }); reply = "thought about it"; + } else if (command0.startsWith("!credential ")) { + const rest = command0.slice("!credential ".length); + const split = rest.indexOf(" "); + const service = split === -1 ? rest : rest.slice(0, split); + const args = split === -1 ? [] : (JSON.parse(rest.slice(split + 1)) as string[]); + if (!turn.tools.credentialExec) throw new Error("credential_exec unavailable"); + await turn.emit({ + type: "tool_call", + payload: { tool: "credential_exec", service, args }, + scopeLabel: turn.scopeLabel, + }); + const result = await turn.tools.credentialExec(service, args); + await turn.emit({ type: "tool_result", payload: result, scopeLabel: turn.scopeLabel }); + turn.onProgress?.({ toolCalls: 1 }); + usedTool = true; + reply = result.stdout.trim() || result.stderr.trim() || `(exit ${result.code})`; } else if (command0.startsWith("!run ") || command0.startsWith("!scratch ") || command0.startsWith("!owner ")) { let tag = "!run "; if (command0.startsWith("!scratch ")) tag = "!scratch "; diff --git a/src/harness/opencode-harness.ts b/src/harness/opencode-harness.ts index df75a8e8d..db90b963a 100644 --- a/src/harness/opencode-harness.ts +++ b/src/harness/opencode-harness.ts @@ -109,7 +109,12 @@ function toolOptions(opts: OpenCodeHarnessOptions, turn?: HarnessTurnInput): PiT backgroundJobTtlMs: opts.backgroundJobTtlMs, backgroundJobTtlMaxMs: opts.backgroundJobTtlMaxMs, ...(turn - ? { readOnly: turn.readOnly, surfaceTools: turn.surfaceTools, surfaceName: turn.surfaceName } + ? { + readOnly: turn.readOnly, + surfaceTools: turn.surfaceTools, + surfaceName: turn.surfaceName, + credentialExecServices: turn.credentialExecServices, + } : { surfaceTools: true, surfaceName: "slack" }), }; } diff --git a/src/harness/pi-harness.ts b/src/harness/pi-harness.ts index 0c8a96b34..5355a8f32 100644 --- a/src/harness/pi-harness.ts +++ b/src/harness/pi-harness.ts @@ -1255,6 +1255,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness { surfaceTools?: boolean, surfaceName?: string, turnScope?: ScopeId, + credentialExecServices?: readonly { service: string; binary: string }[], tapeRows?: TapeRecord[], tapeMode?: "shadow" | "serve", tapeFold?: unknown[], @@ -1314,6 +1315,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness { ownerAuthExec, reachExec, controlTools, + ...(credentialExecServices?.length ? { credentialExecServices } : {}), ...(surfaceTools ? { surfaceTools: true } : {}), ...(surfaceName ? { surfaceName } : {}), ...(readOnly ? { readOnly: true } : {}), @@ -1470,6 +1472,7 @@ export function createPiHarness(opts?: PiHarnessOptions): Harness { turn.surfaceTools, turn.surfaceName, turn.scopeLabel, + turn.credentialExecServices, turn.tapeRows, turn.tapeMode, turn.tapeFold, diff --git a/src/harness/pi-tools.ts b/src/harness/pi-tools.ts index 778772611..256ae3c55 100644 --- a/src/harness/pi-tools.ts +++ b/src/harness/pi-tools.ts @@ -121,6 +121,44 @@ function capText(t: string): string { return t.length > MAX_TOOL_RESULT_CHARS ? `${t.slice(0, MAX_TOOL_RESULT_CHARS)}…[truncated]` : t; } +function contentFactLines(content: string): string[] { + return content + .split(/\r?\n/) + .map((line) => + line + .trim() + .replace(/^[-*]\s+/, "") + .trim(), + ) + .filter(Boolean); +} + +function passedRememberFields(params: { facts?: unknown; content?: unknown; query?: unknown }): string { + const fields = ["facts", "content", "query"].filter((field) => params[field as keyof typeof params] !== undefined); + return fields.length ? fields.join(", ") : "none"; +} + +function rememberFacts(params: { facts?: unknown; content?: unknown; query?: unknown }): { + facts: string[]; + coercedFrom?: "facts" | "content" | "query"; +} { + if (typeof params.facts === "string") { + const fact = params.facts.trim(); + if (fact) return { facts: [fact], coercedFrom: "facts" }; + } + const facts = Array.isArray(params.facts) ? params.facts.map((fact) => String(fact).trim()).filter(Boolean) : []; + if (facts.length) return { facts }; + if (typeof params.content === "string") { + const contentFacts = contentFactLines(params.content); + if (contentFacts.length) return { facts: contentFacts, coercedFrom: "content" }; + } + if (typeof params.query === "string") { + const fact = params.query.trim(); + if (fact) return { facts: [fact], coercedFrom: "query" }; + } + return { facts: [] }; +} + function fmtStatus(s: { state: "running" } | { state: "exited"; code: number }): string { return s.state === "exited" ? `exited ${s.code}` : "running"; } @@ -230,6 +268,7 @@ function fmtCronRunLine(entry: CronFireLogEntry): string { } export interface PiToolsOptions { + credentialExecServices?: readonly { service: string; binary: string }[]; scratchExec?: boolean; ownerAuthExec?: boolean; reachExec?: boolean; @@ -279,6 +318,7 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolD const ownerAuthExec = !!opts?.ownerAuthExec; const reachExec = !!opts?.reachExec; const controlTools = !!opts?.controlTools; + const credentialExecServices = opts?.credentialExecServices ?? ref.current?.credentialExecServices ?? []; const surfaceTools = !!opts?.surfaceTools; const execTimeoutSec = Math.round((opts?.execTimeoutMs ?? CONFIG_DEFAULTS.execTimeoutDefaultSec * 1000) / 1000); const execCeilingSec = Math.round((opts?.execTimeoutCeilingMs ?? CONFIG_DEFAULTS.execTimeoutMaxSec * 1000) / 1000); @@ -892,7 +932,7 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolD ...(params.query !== undefined ? { query: params.query } : {}), ...(params.limit !== undefined ? { limit: params.limit } : {}), ...(params.facts !== undefined ? { facts: params.facts } : {}), - ...(params.content !== undefined ? { chars: params.content.length } : {}), + ...(typeof params.content === "string" ? { chars: params.content.length } : {}), }); const unavailable = () => recordResult( @@ -937,19 +977,24 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolD ); } case "remember": { - const facts = (params.facts ?? []).map((f) => f.trim()).filter(Boolean); + const resolved = rememberFacts(params); + const { facts } = resolved; if (!facts.length) return recordResult( callId, { tool: "memory", action, error: "facts required" }, - text("[error] memory remember requires `facts` (a non-empty list)."), + text( + `[error] memory remember requires \`facts\` (a non-empty list). Received: ${passedRememberFields( + params, + )} (use facts instead).`, + ), true, ); const added = await tc.memoryRemember(facts); if (added === null) return unavailable(); return recordResult( callId, - { tool: "memory", action, added }, + { tool: "memory", action, added, ...(resolved.coercedFrom ? { coercedFrom: resolved.coercedFrom } : {}) }, text( added ? `Remembered ${added} fact${added === 1 ? "" : "s"}.` @@ -1209,6 +1254,20 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolD ...(params.pattern !== undefined ? { pattern: params.pattern } : {}), ...(params.since_cursor !== undefined ? { sinceCursor: params.since_cursor } : {}), }); + if ("completed" in r) { + const status = `${r.registryStatus}${r.exitCode !== undefined ? ` (code ${r.exitCode})` : ""}`; + const result = r.outputTail + ? `job already ${status} — no watch armed; here is the tail of its output:\n${r.outputTail}` + : `job already ${status} — no watch armed; it produced no output.`; + return recordResult( + callId, + { tool: "background", ...r }, + { + content: [{ type: "text" as const, text: result }], + details: r, + }, + ); + } const trigger = params.pattern ? `new output matching /${params.pattern}/` : "new output"; return recordResult( callId, @@ -1742,6 +1801,12 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolD action: Type.Union([Type.Literal("read"), Type.Literal("write")]), scope: Type.Optional(Type.Union([Type.Literal("channel"), Type.Literal("conversation")])), content: Type.Optional(Type.String()), + ambientEnabled: Type.Optional( + Type.Union([Type.Boolean(), Type.Null()], { + description: + "Channel scope only. true judges every message for an unprompted reply, false responds only when addressed, and null uses the platform default.", + }), + ), bots: Type.Optional( Type.Record( Type.String(), @@ -1796,24 +1861,39 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolD const note = convoExists ? "\n\n(conversation-scope guidance also exists — read with scope=conversation)" : ""; - const body = (channel!.orders.trim() ? channel!.orders : "[no channel guidance set]") + ledger + note; + let ambientState = "default"; + if (channel!.ambientEnabled !== undefined) ambientState = channel!.ambientEnabled ? "on" : "off"; + const ambient = `\n\nAmbient replies: ${ambientState}`; + const body = + (channel!.orders.trim() ? channel!.orders : "[no channel guidance set]") + ambient + ledger + note; return recordResult(callId, { tool: "guidance", scope, ok: true }, text(body)); } - if (typeof params.content !== "string" && params.bots === undefined) { + if (typeof params.content !== "string" && params.bots === undefined && params.ambientEnabled === undefined) { return recordResult( callId, - { tool: "guidance", scope, error: "content or bots required" }, - text("[error] guidance write needs `content` (the full new channel guidance) and/or `bots`."), + { tool: "guidance", scope, error: "content, bots, or ambientEnabled required" }, + text( + "[error] guidance write needs `content` (the full new channel guidance), `bots`, and/or `ambientEnabled`.", + ), true, ); } const orders = typeof params.content === "string" ? params.content : channel!.orders; - const r = await tc.setStandingOrder(orders, params.bots); + const r = await tc.setStandingOrder(orders, params.bots, params.ambientEnabled); if (!r.ok) return recordResult(callId, { tool: "guidance", scope, ok: false }, text(`[error] ${r.message}`), true); return recordResult(callId, { tool: "guidance", scope, ok: true }, text("[channel guidance updated]")); } + if (params.ambientEnabled !== undefined) { + return recordResult( + callId, + { tool: "guidance", scope, error: "ambientEnabled is channel scope only" }, + text("[error] `ambientEnabled` applies only to channel scope."), + true, + ); + } + if (params.action === "read") { const r = tc.soulRead(); if (isUnavailable(r)) return unavailable(callId, "guidance"); @@ -2394,8 +2474,78 @@ export function createPiTools(ref: ToolContextRef, opts?: PiToolsOptions): ToolD }, }); + const credentialExec = defineTool({ + name: "credential_exec", + label: "Credential exec", + description: + "Run a configured credential-bearing CLI in a one-shot isolated box. Shell operators and pipelines are not supported; args are passed literally. Available services: " + + credentialExecServices.map(({ service, binary }) => `${service} (${binary})`).join(", "), + parameters: Type.Object({ + service: Type.String({ enum: credentialExecServices.map(({ service }) => service) }), + args: Type.Array(Type.String()), + timeout_seconds: Type.Optional(Type.Integer({ minimum: 1, maximum: execCeilingSec })), + }), + async execute(callId, params: { service: string; args: string[]; timeout_seconds?: number }) { + const tc = ref.current; + await recordCall(callId, { tool: "credential_exec", service: params.service, args: params.args }); + if (!tc?.credentialExec) { + return recordResult( + callId, + { tool: "credential_exec", unavailable: true }, + text("[error] credential_exec is unavailable on this turn"), + true, + ); + } + try { + const result = await tc.credentialExec(params.service, params.args, { + ...(params.timeout_seconds !== undefined ? { timeoutSeconds: params.timeout_seconds } : {}), + ...(ref.abortSignal ? { signal: ref.abortSignal } : {}), + }); + const parts = [result.stdout, result.stderr ? `[stderr]\n${result.stderr}` : ""].filter(Boolean).join("\n"); + return recordResult( + callId, + { tool: "credential_exec", service: params.service, ...result }, + text(`${parts}\n[exit ${result.code}${result.timedOut ? " timed-out" : ""}]`), + result.code !== 0, + ); + } catch (error) { + if (error instanceof NeedsApproval) { + ref.pendingApprovals?.push({ + command: error.command, + reason: error.approvalReason, + kind: error.kind, + matched: error.matched, + ...(error.approvalKey ? { approvalKey: error.approvalKey } : {}), + }); + ref.pausedOnApproval = true; + return recordResult( + callId, + { tool: "credential_exec", blocked: "needs_approval", reason: error.approvalReason }, + { ...text(`[blocked: needs human approval] ${error.approvalReason}`), terminate: true }, + true, + ); + } + if (error instanceof CommandDenied) { + return recordResult( + callId, + { tool: "credential_exec", denied: true, reason: error.message }, + text(`[denied by policy] ${error.message}`), + true, + ); + } + return recordResult( + callId, + { tool: "credential_exec", service: params.service, failed: true }, + text(`[error] ${errMessage(error)}`), + true, + ); + } + }, + }); + const tools = [ execute, + ...(credentialExecServices.length ? [credentialExec] : []), read, write, publish, diff --git a/src/monitors/monitor-broker.ts b/src/monitors/monitor-broker.ts index 08e7251d2..9bc1df38c 100644 --- a/src/monitors/monitor-broker.ts +++ b/src/monitors/monitor-broker.ts @@ -1,14 +1,31 @@ import type { Destination, Monitor, ScopeId } from "../types.ts"; import type { MonitorStore } from "./monitor-store.ts"; -import type { ProcessRegistry } from "../processes/process-registry.ts"; +import type { ProcessRegistry, ProcessStatus } from "../processes/process-registry.ts"; -export interface BackgroundWatchResult { +export interface BackgroundWatchArmedResult { monitorId: string; processId: string; reattached: boolean; expiresAt: number; } +interface BackgroundWatchCompletedResult { + processId: string; + completed: true; + registryStatus: ProcessStatus; + exitCode?: number; + outputTail: string; + cursor?: number; +} + +export type BackgroundWatchResult = BackgroundWatchArmedResult | BackgroundWatchCompletedResult; + +export interface BackgroundOutputTail { + outputTail: string; + cursor?: number; + exitCode?: number; +} + export interface BackgroundUnwatchResult { monitorId: string; removed: boolean; @@ -25,6 +42,33 @@ export interface MonitorBroker { const EXPIRY_GRACE_MS = 5 * 60_000; const MAX_PATTERN_CHARS = 256; +const WATCH_COMPLETED_TAIL_BYTES = 4 * 1024; + +export async function readBackgroundOutputTail( + maxBytes: number, + readNext: (cursor: number, maxBytes: number) => Promise<{ chunks: string; cursor: number; exitCode?: number }>, +): Promise { + let cursor = 0; + let outputTail = ""; + let exitCode: number | undefined; + for (;;) { + const read = await readNext(cursor, maxBytes); + if (read.exitCode !== undefined) exitCode = read.exitCode; + if (read.chunks) { + const bytes = Buffer.from(outputTail + read.chunks); + outputTail = + bytes.length > maxBytes ? bytes.subarray(bytes.length - maxBytes).toString("utf8") : bytes.toString("utf8"); + } + if (!read.chunks || read.cursor === cursor) { + return { + outputTail, + cursor: read.cursor, + ...(exitCode !== undefined ? { exitCode } : {}), + }; + } + cursor = read.cursor; + } +} export function compileMonitorPattern(pattern: string): (line: string) => boolean { if (!pattern || pattern.length > MAX_PATTERN_CHARS) @@ -52,6 +96,7 @@ export function compileMonitorPattern(pattern: string): (line: string) => boolea export interface MonitorBrokerDeps { store: MonitorStore; registry: ProcessRegistry; + readOutputTail(processId: string, maxBytes: number): Promise; scopeId: string; owner: string; ownerScopeId: ScopeId; @@ -69,7 +114,15 @@ export function createMonitorBroker(deps: MonitorBrokerDeps): MonitorBroker { throw new Error("no such background job to watch"); } if (rec.status !== "running") { - throw new Error(`background job already ${rec.status} — nothing left to watch`); + const tail = await deps.readOutputTail(processId, WATCH_COMPLETED_TAIL_BYTES); + return { + processId, + completed: true, + registryStatus: rec.status, + ...(tail.exitCode !== undefined ? { exitCode: tail.exitCode } : {}), + outputTail: tail.outputTail, + ...(tail.cursor !== undefined ? { cursor: tail.cursor } : {}), + }; } if (opts?.pattern !== undefined) { try { diff --git a/src/policy/command-policy.ts b/src/policy/command-policy.ts index b68103c9e..ed60fa16d 100644 --- a/src/policy/command-policy.ts +++ b/src/policy/command-policy.ts @@ -87,7 +87,24 @@ function scannableCommandAtDepth(command: string, depth: number): string { function stripWrittenHeredocs(command: string): string { return command.replace( /^([^\n]*)<<-?\s*(["']?)([A-Za-z_]\w*)\2([^\n]*)\n([\s\S]*?)^\s*\3\s*$/gm, - (full, pre, _q, _delim, post) => (/[>]/.test(pre + post) && !heredocRunsShell(pre + post) ? "" : full), + (full, pre, quote, _delim, post, body) => { + if (heredocRunsInterpreter(pre + post)) return full; + // The heredoc body is data (written to a file or fed to a non-interpreter + // command like cat/gh/jq), not something the shell executes. Keep only + // command substitutions, which DO execute when the delimiter is unquoted. + if (quote) return ""; + const subs = (body as string).match(/\$\([^)]*\)|`[^`]*`/g); + return subs ? subs.join(" ") : ""; + }, + ); +} + +function heredocRunsInterpreter(commandLine: string): boolean { + if (heredocRunsShell(commandLine)) return true; + // SQL clients and script interpreters execute their stdin, so a heredoc fed + // to them must stay visible to the rules (e.g. destructive SQL via psql). + return /(?:^|[|;&]\s*)(?:\S*\/)?(?:psql|mysql|mariadb|sqlite3?|sqlcmd|python\d?(?:\.\d+)?|node|perl|ruby)\b/.test( + commandLine, ); } diff --git a/src/runs/worker.ts b/src/runs/worker.ts index f257afde6..818b5f038 100644 --- a/src/runs/worker.ts +++ b/src/runs/worker.ts @@ -68,7 +68,9 @@ export async function processRun(deps: ProcessDeps, run: Run, opts?: { backgroun ...(queueMs !== undefined ? { queueMs } : {}), }); stopBeat(); - await deps.runs.complete(run.id, token, result); + if (!(await deps.runs.complete(run.id, token, result))) { + throw new Error(`run ${run.id} lost its lease before completion`); + } return result; } catch (err) { stopBeat(); diff --git a/src/sandbox/sprites-sandbox.ts b/src/sandbox/sprites-sandbox.ts index dbfdbcdf8..f7ddb03bc 100644 --- a/src/sandbox/sprites-sandbox.ts +++ b/src/sandbox/sprites-sandbox.ts @@ -59,7 +59,6 @@ export interface SpritesSandboxOptions { signingSecret?: string; capabilitySecret?: string; apiBaseUrl?: string; - extraTools?: string[]; credentialPaths?: CredentialPathSpec[]; client?: SpritesClientLike; fetchImpl?: typeof fetch; @@ -310,10 +309,10 @@ export function createSpritesSandbox(workspace: WorkspaceStore, opts: SpritesSan os: "Ubuntu 26.04 LTS — Fly Sprite microVM (auto-sleeps when idle; the whole disk persists)", runtimes: ["Node 24", "Python 3"], get tools() { - return visibleTools(["git", "curl", "jq", "tar", "python3", ...(opts.extraTools ?? [])]); + return visibleTools(["git", "curl", "jq", "tar", "python3"]); }, get notInstalled() { - return visibleNotInstalled(["gh", "aws", "gcloud", "kubectl", "flyctl", "glab"], opts.extraTools ?? []); + return visibleNotInstalled(["gh", "aws", "gcloud", "kubectl", "flyctl", "glab"], []); }, diskGb: 100, homeDir: HOME_DIR, diff --git a/src/sessions/memory-session-store.ts b/src/sessions/memory-session-store.ts index f1f1986ce..8869b5c68 100644 --- a/src/sessions/memory-session-store.ts +++ b/src/sessions/memory-session-store.ts @@ -107,6 +107,7 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStore }, async acquireLease(sessionId, holder): Promise { + if (!sessions.has(sessionId)) return { lease: null }; const held = leases.get(sessionId); if (held && now() < held.expiresAt) return { @@ -143,6 +144,15 @@ export function createMemorySessionStore(opts: StoreOptions = {}): SessionStore leases.delete(sessionId); }, + async deleteSessionIfEmpty(sessionId) { + if (!sessions.has(sessionId)) return false; + if ((entries.get(sessionId)?.length ?? 0) > 0) return false; + const held = leases.get(sessionId); + if (held && now() < held.expiresAt) return false; + await this.deleteSession(sessionId); + return true; + }, + async forceReleaseLease(sessionId) { leases.delete(sessionId); }, diff --git a/src/sessions/postgres-session-store.ts b/src/sessions/postgres-session-store.ts index 75942126b..a6917cfbd 100644 --- a/src/sessions/postgres-session-store.ts +++ b/src/sessions/postgres-session-store.ts @@ -262,8 +262,12 @@ export function createPostgresSessionStore(connectionString: string, opts: Store OR ${lastActivityExpr("s")} > (EXTRACT(EPOCH FROM now()) * 1000)::bigint - 172800000`, ]); + const lockSession = (client: PoolClient, sessionId: string) => + client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [sessionId]); + const withLease = async (lease: Lease, invalidMsg: string, fn: (client: PoolClient) => Promise): Promise => withPgTransaction(await pool(), async (client) => { + await lockSession(client, lease.sessionId); const held = await client.query("SELECT token FROM session_leases WHERE session_id = $1 FOR UPDATE", [ lease.sessionId, ]); @@ -371,27 +375,31 @@ export function createPostgresSessionStore(connectionString: string, opts: Store async acquireLease(sessionId, holder): Promise { const token = randomUUID(); const t = now(); - const rows = await q( - `INSERT INTO session_leases(session_id, token, expires_at, holder, acquired_at) - VALUES ($1,$2,$3,$5,$4) - ON CONFLICT (session_id) DO UPDATE - SET token = $2, expires_at = $3, holder = $5, acquired_at = $4 - WHERE session_leases.expires_at <= $4 - RETURNING token`, - [sessionId, token, t + leaseTtlMs, t, holder ?? null], - ); - if (rows[0]) return { lease: { sessionId, token } }; - const held = await q("SELECT expires_at, holder, acquired_at FROM session_leases WHERE session_id = $1", [ - sessionId, - ]); - const row = held[0]; - if (!row) return { lease: null }; - return { - lease: null, - ...(row.holder != null ? { heldBy: row.holder as LeaseHolder } : {}), - ...(row.acquired_at != null ? { heldSince: Number(row.acquired_at) } : {}), - heldUntil: Number(row.expires_at), - }; + return withPgTransaction(await pool(), async (client) => { + await lockSession(client, sessionId); + const granted = await client.query( + `INSERT INTO session_leases(session_id, token, expires_at, holder, acquired_at) + SELECT $1, $2, $3, $5, $4 WHERE EXISTS (SELECT 1 FROM sessions WHERE id = $1) + ON CONFLICT (session_id) DO UPDATE + SET token = $2, expires_at = $3, holder = $5, acquired_at = $4 + WHERE session_leases.expires_at <= $4 + RETURNING token`, + [sessionId, token, t + leaseTtlMs, t, holder ?? null], + ); + if (granted.rows[0]) return { lease: { sessionId, token } }; + const held = await client.query( + "SELECT expires_at, holder, acquired_at FROM session_leases WHERE session_id = $1", + [sessionId], + ); + const row = held.rows[0]; + if (!row) return { lease: null }; + return { + lease: null, + ...(row.holder != null ? { heldBy: row.holder as LeaseHolder } : {}), + ...(row.acquired_at != null ? { heldSince: Number(row.acquired_at) } : {}), + heldUntil: Number(row.expires_at), + }; + }); }, async releaseLease(lease): Promise { @@ -600,6 +608,7 @@ export function createPostgresSessionStore(connectionString: string, opts: Store async deleteSession(sessionId): Promise { await withPgTransaction(await pool(), async (client) => { + await lockSession(client, sessionId); await client.query("DELETE FROM session_llm_requests WHERE session_id = $1", [sessionId]); await client.query("DELETE FROM session_leases WHERE session_id = $1", [sessionId]); await client.query("DELETE FROM participants WHERE session_id = $1", [sessionId]); @@ -609,6 +618,25 @@ export function createPostgresSessionStore(connectionString: string, opts: Store }); }, + async deleteSessionIfEmpty(sessionId): Promise { + return withPgTransaction(await pool(), async (client) => { + await lockSession(client, sessionId); + const gone = await client.query( + `DELETE FROM sessions + WHERE id = $1 + AND NOT EXISTS (SELECT 1 FROM session_entries WHERE session_id = $1) + AND NOT EXISTS (SELECT 1 FROM session_leases WHERE session_id = $1 AND expires_at > $2)`, + [sessionId, now()], + ); + if (gone.rowCount === 0) return false; + await client.query("DELETE FROM session_llm_requests WHERE session_id = $1", [sessionId]); + await client.query("DELETE FROM session_leases WHERE session_id = $1", [sessionId]); + await client.query("DELETE FROM participants WHERE session_id = $1", [sessionId]); + await client.query("DELETE FROM session_tape WHERE session_id = $1", [sessionId]); + return true; + }); + }, + async listByParticipant(principalId): Promise { const rows = await q( `SELECT s.*, p.title AS p_title, p.archived AS p_archived, p.pinned AS p_pinned, p.color AS p_color, diff --git a/src/sessions/session-store.ts b/src/sessions/session-store.ts index 6f056d525..89bd25497 100644 --- a/src/sessions/session-store.ts +++ b/src/sessions/session-store.ts @@ -440,6 +440,7 @@ export interface SessionStore { listByParticipant(principalId: string): Promise; deleteSession(sessionId: string): Promise; + deleteSessionIfEmpty(sessionId: string): Promise; updateParticipantView(sessionId: string, principalId: string, patch: ParticipantViewPatch): Promise; diff --git a/src/tools/primitives.ts b/src/tools/primitives.ts index e764278e5..596cc7935 100644 --- a/src/tools/primitives.ts +++ b/src/tools/primitives.ts @@ -144,6 +144,12 @@ interface ReachedProvenance { } export interface ToolContext extends SurfaceToolDeps { + credentialExecServices?: readonly { service: string; binary: string }[]; + credentialExec?( + service: string, + args: string[], + opts?: { timeoutSeconds?: number; signal?: AbortSignal }, + ): Promise; execute( command: string, opts?: { @@ -305,7 +311,8 @@ interface SurfaceFileResult { } type SurfaceStandingOrderResult = - { ok: true; orders: string; bots?: Record } | { ok: false; message: string }; + | { ok: true; orders: string; bots?: Record; ambientEnabled?: boolean } + | { ok: false; message: string }; export interface SurfaceToolDeps { post(text: string, opts?: SurfacePostOpts, files?: readonly string[]): Promise; @@ -319,7 +326,11 @@ export interface SurfaceToolDeps { readMembers(): Promise; readFile(ref: string): Promise; getStandingOrder(): Promise; - setStandingOrder(orders: string, bots?: Record): Promise; + setStandingOrder( + orders: string, + bots?: Record, + ambientEnabled?: boolean | null, + ): Promise; staySilent(reason: string): Promise<{ ok: true; message: string }>; } @@ -336,10 +347,13 @@ export const CONTROL_UNAVAILABLE: ControlUnavailable = { export interface ToolContextDeps { sandbox: Sandbox; + credentialExecServices?: readonly { service: string; binary: string }[]; + credentialExec?: ToolContext["credentialExec"]; provision: () => Promise; provisionScratch?: () => Promise; provisionOwnerAuth?: () => Promise; ownerAuthCommand?: (command: string) => string; + scopedCommand?: (command: string) => string; ensureSkillTree?: (skillDir: string) => Promise; reach?: { resolveChannel(query: string): Promise; @@ -443,6 +457,8 @@ export function createToolContext(deps: ToolContextDeps): ToolContext { } return { + ...(deps.credentialExecServices ? { credentialExecServices: deps.credentialExecServices } : {}), + ...(deps.credentialExec ? { credentialExec: deps.credentialExec } : {}), async execute( command: string, execOpts?: { @@ -519,7 +535,9 @@ export function createToolContext(deps: ToolContextDeps): ToolContext { }); } return timed("exec", async () => { - const sandboxCommand = ownerAuth && deps.ownerAuthCommand ? deps.ownerAuthCommand(command) : command; + const sandboxCommand = ownerAuth + ? (deps.ownerAuthCommand?.(command) ?? command) + : (deps.scopedCommand?.(command) ?? command); const r = await deps.sandbox.run(handle, sandboxCommand, opts); return reached ? { ...r, reached } : r; }); @@ -803,7 +821,12 @@ export function createToolContext(deps: ToolContextDeps): ToolContext { for (const skillDir of skillTreeDirsInCommand(command)) await deps.ensureSkillTree(skillDir); } return once( - () => deps.backgroundBroker!.start(handle, command, opts?.ttlSeconds ? opts.ttlSeconds * 1000 : undefined), + () => + deps.backgroundBroker!.start( + handle, + deps.scopedCommand?.(command) ?? command, + opts?.ttlSeconds ? opts.ttlSeconds * 1000 : undefined, + ), () => true, ); }, @@ -943,7 +966,8 @@ export function createToolContext(deps: ToolContextDeps): ToolContext { readMembers: () => surfaceOp((s) => s.readMembers()), readFile: (ref) => surfaceOp((s) => s.readFile(ref)), getStandingOrder: () => surfaceOp((s) => s.getStandingOrder()), - setStandingOrder: (orders, bots) => surfaceOp((s) => s.setStandingOrder(orders, bots)), + setStandingOrder: (orders, bots, ambientEnabled) => + surfaceOp((s) => s.setStandingOrder(orders, bots, ambientEnabled)), staySilent: (reason) => deps.surface ? deps.surface.staySilent(reason) diff --git a/src/triggers/run-trigger.ts b/src/triggers/run-trigger.ts index 33c1f4926..9da9eae50 100644 --- a/src/triggers/run-trigger.ts +++ b/src/triggers/run-trigger.ts @@ -12,7 +12,7 @@ import type { IdentityService } from "../identity/identity-service.ts"; import type { DeliveryStore } from "../delivery/delivery-store.ts"; import type { IdempotencyStore } from "../idempotency/idempotency-store.ts"; import { turnModelOptions } from "../core/turn-options.ts"; -import { reachEnqueue } from "../reach/reach.ts"; +import { principalDestination, reachEnqueue } from "../reach/reach.ts"; import { consentRequiredRecipient, recipientConsentSatisfied } from "./trigger-store.ts"; import { isVisible, type VisibilityDirectory } from "../directory/visibility.ts"; import { samePerson } from "../directory/person.ts"; @@ -188,15 +188,27 @@ export async function runTrigger(deps: TriggerDeps, spec: TriggerSpec): Promise< let note: string | undefined; let reply: string | undefined; let sessionId: string | undefined; + const ownerSkipNotice = async () => { + await deps.deliveries.enqueue({ + destination: principalDestination(spec.owner, spec.owner), + text: `Scheduled delivery skipped: ${consentNote}`, + idempotencyKey: `${spec.fireKey}:err`, + provenance: deliveryProvenance(spec, threadRef), + ...(spec.shadow ? { shadow: true } : {}), + }); + }; const ran = await deps.idempotency.once(spec.fireKey, async () => { if (spec.message !== undefined) { status = "ok"; if (!spec.destination) return; if (!consented) { + status = "refused"; note = consentNote; + await ownerSkipNotice(); return; } if (!deliverable) { + status = "refused"; note = notVisibleNote; return; } @@ -248,10 +260,13 @@ export async function runTrigger(deps: TriggerDeps, spec: TriggerSpec): Promise< if (!spec.destination) return; if (liveDelivery) return; if (!consented) { + status = "refused"; note = consentNote; + await ownerSkipNotice(); return; } if (!deliverable) { + status = "refused"; note = notVisibleNote; return; } diff --git a/src/types.ts b/src/types.ts index ab5319b37..ae5968b2b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -217,6 +217,7 @@ export interface CronFireLogEntry { export interface Cron extends TriggerBase { schedule: CronSchedule; nextFireAt?: number; + lastAttemptAt?: number; title?: string; archived?: boolean; action?: string; diff --git a/src/wiring.ts b/src/wiring.ts index 83540e3d8..6355bed91 100644 --- a/src/wiring.ts +++ b/src/wiring.ts @@ -259,6 +259,7 @@ import { createDeploymentLayerStore, LAYER_CREATED_BY, LAYER_REVIEWER, + type DeploymentLayerBundle, type DeploymentLayerStore, type StoredDeploymentLayer, } from "./deployment/deployment-layer-store.ts"; @@ -461,6 +462,10 @@ export function buildApp( const deploymentLayer = config.deploymentLayerDir ? loadDeploymentLayer(config.deploymentLayerDir) : emptyDeploymentLayer(); + const spritesEnabled = config.sandboxBackend === "sprites" || config.sandboxSecondaryBackend === "sprites"; + if (spritesEnabled && deploymentLayer.tools.length) { + throw new Error("SANDBOX_BACKEND=sprites does not support deployment-layer tools; deploy skills only"); + } const layerSkillsDir = config.deploymentLayerDir ? resolve(deploymentLayer.dir, "skills") : undefined; const brokeredTools = deploymentLayer.brokeredTools; const orgScope = scopeId("org", config.orgId); @@ -468,6 +473,15 @@ export function buildApp( const deploymentLayerStore = createDeploymentLayerStore({ backing: artifactMap("deployment_layer"), runtime: deploymentLayer, + ...(spritesEnabled + ? { + validateBundle: (bundle: DeploymentLayerBundle): void => { + if (bundle.tools.length) { + throw new Error("SANDBOX_BACKEND=sprites does not support deployment-layer tools; deploy skills only"); + } + }, + } + : {}), skills, skillBundles, scopeId: orgScope, @@ -576,7 +590,6 @@ export function buildApp( createSpritesSandbox(workspace, { ...config.spritesSandbox, blobTransfer, - extraTools: deploymentLayer.advertisedTools, credentialPaths: deploymentLayer.credentialPaths, ...(config.signingSecret ? { signingSecret: config.signingSecret } : {}), ...(config.capabilitySecret ? { capabilitySecret: config.capabilitySecret } : {}), diff --git a/test/admin-observability.test.ts b/test/admin-observability.test.ts index 529a498b0..f8db7c4c3 100644 --- a/test/admin-observability.test.ts +++ b/test/admin-observability.test.ts @@ -84,7 +84,7 @@ test("an org admin sees conversations, transcripts, files, and runs top-down", a assert.equal(download.status, 200); assert.equal( download.headers.get("content-type"), - "text/plain", + "text/plain; charset=utf-8", "browser-renderable types open in the browser instead of downloading", ); assert.match(download.headers.get("content-disposition") ?? "", /^inline/); diff --git a/test/admin-scopes-directory.test.ts b/test/admin-scopes-directory.test.ts index dcb280e50..b604750af 100644 --- a/test/admin-scopes-directory.test.ts +++ b/test/admin-scopes-directory.test.ts @@ -16,6 +16,8 @@ function start() { admin: built.admin, auditLog: built.auditLog, sessions: built.sessions, + config: built.config, + environments: built.environments, }); server.listen(0); const base = `http://localhost:${(server.address() as AddressInfo).port}`; @@ -118,3 +120,34 @@ test("scopes without a session label fall back to the org directory (people's na await s.close(); } }); + +test("the admin scope directory exposes named environments and attached scopes", async () => { + const s = start(); + try { + await s.built.app.upsertChannels([ + { channelId: "A", name: "source" }, + { channelId: "B", name: "attached" }, + ]); + await s.built.app.createEnvironment({ scopeId: "channel:A", name: "A-permanent", actorId: "U1" }); + await s.built.app.attachScope({ scopeId: "channel:B", environmentId: "channel:A", actorId: "U1" }); + + const directory = await json(await fetch(`${s.base}/v1/admin/scopes`, { headers: ALICE_ADMIN })); + const byId = new Map(directory.scopes.map((row: any) => [row.scopeId, row])); + assert.deepEqual(directory.environments, [ + { id: "channel:A", name: "A-permanent", ownerActorId: "U1", attachedScopes: ["channel:B"] }, + ]); + assert.equal((byId.get("channel:A") as any).environmentName, "A-permanent"); + assert.deepEqual((byId.get("channel:B") as any).environmentAttachment, { + environmentId: "channel:A", + environmentName: "A-permanent", + }); + + const attached = await json(await fetch(`${s.base}/v1/admin/scopes/channel:B`, { headers: ALICE_ADMIN })); + assert.deepEqual(attached.environmentAttachment, { + environmentId: "channel:A", + environmentName: "A-permanent", + }); + } finally { + await s.close(); + } +}); diff --git a/test/agent-computer-profile.test.ts b/test/agent-computer-profile.test.ts index f2a933562..08881d0ba 100644 --- a/test/agent-computer-profile.test.ts +++ b/test/agent-computer-profile.test.ts @@ -47,6 +47,7 @@ test("the sprites sandbox declares the Agent Computer contract (persistent per-s assert.match(spec?.os ?? "", /Ubuntu/); assert.equal(spec?.homeDir, "/home/sprite"); assert.equal(spec?.workdir, "/home/sprite/workspace"); + assert.deepEqual(spec?.tools, ["git", "curl", "jq", "tar", "python3"]); assert.ok(spec?.notInstalled?.includes("gh")); assert.equal(typeof sprites.backupComputer, "function"); }); diff --git a/test/agent-conversations-route.test.ts b/test/agent-conversations-route.test.ts index ba06956d3..10eaadba9 100644 --- a/test/agent-conversations-route.test.ts +++ b/test/agent-conversations-route.test.ts @@ -85,6 +85,65 @@ describe("agent conversations self-API", async () => { ); }); + it("spawns a fresh channel conversation for a current member", async () => { + await built.app.upsertDirectory([{ principalId: "U1", displayName: "User One", type: "internal" }]); + await built.app.upsertChannels( + [{ channelId: "C1", name: "engineering", isPrivate: true }], + [{ channelId: "C1", principalId: "U1" }], + ); + const res = await post( + "/v1/conversations", + { text: "investigate the channel deployment" }, + await capFor("U1", scopeId("channel", "C1")), + ); + assert.equal(res.status, 202); + const body = (await res.json()) as { + session: { scopeId: string }; + turn: { status: string; runId?: string }; + }; + assert.equal(body.session.scopeId, scopeId("channel", "C1")); + assert.equal(body.turn.status, "queued"); + assert.ok(body.turn.runId); + const run = await built.runs.get(body.turn.runId); + assert.equal(run?.request.conversation.channelRef, "C1"); + }); + + it("discards the spawned session when the seed turn is refused (roster race)", async () => { + const racedApp: typeof built.app = { + ...built.app, + turn: async (req) => + (req as { spawned?: boolean }).spawned + ? { status: "refused", reason: "project membership changed; retry from the current project" } + : built.app.turn(req), + }; + const racedServer = createServer(racedApp, { signingSecret: SECRET }); + await new Promise((resolve) => racedServer.listen(0, resolve)); + const racedBase = `http://localhost:${(racedServer.address() as AddressInfo).port}`; + try { + const token = await capFor("U1"); + const listIds = async () => { + const listed = await get("/v1/conversations", token); + const { conversations } = (await listed.json()) as { conversations: Array<{ id: string }> }; + return conversations.map((c) => c.id).sort(); + }; + const before = await listIds(); + + const res = await fetch(`${racedBase}/v1/conversations`, { + method: "POST", + headers: { "content-type": "application/json", "x-agent-capability": token }, + body: JSON.stringify({ text: "seed that will be refused" }), + }); + assert.equal(res.status, 409); + const body = (await res.json()) as { error: string; message: string }; + assert.equal(body.error, "seed_turn_refused"); + assert.match(body.message, /membership changed/); + + assert.deepEqual(await listIds(), before, "no orphaned empty session survives the refused seed"); + } finally { + await new Promise((resolve) => racedServer.close(() => resolve())); + } + }); + it("spawn requires text and a capability", async () => { assert.equal((await post("/v1/conversations", { text: "hi" })).status, 401); assert.equal((await post("/v1/conversations", {}, await capFor("U1"))).status, 400); diff --git a/test/agent-files-route.test.ts b/test/agent-files-route.test.ts index a60fc371a..958fe5638 100644 --- a/test/agent-files-route.test.ts +++ b/test/agent-files-route.test.ts @@ -6,6 +6,7 @@ import type { AddressInfo } from "node:net"; import type { Server } from "node:http"; import { buildApp, type BuiltApp } from "../src/wiring.ts"; import { createServer } from "../src/api/server.ts"; +import { contentTypeWithUtf8Charset } from "../src/api/http.ts"; import { fileArtifactId } from "../src/files/file-artifact-store.ts"; import { mintCapabilityToken, CAPABILITY_TTL_MS, CONTROL_PLANE_AUD } from "../src/auth/capability-token.ts"; import { scopeId } from "../src/types.ts"; @@ -13,12 +14,30 @@ import { testConfig } from "./support/test-config.ts"; const SECRET = "agent-files-secret".repeat(3); +it("detects charset parameters outside quoted parameter values", () => { + assert.equal( + contentTypeWithUtf8Charset('text/plain; note="x;charset=bogus"'), + 'text/plain; note="x;charset=bogus"; charset=utf-8', + ); + assert.equal( + contentTypeWithUtf8Charset("text/plain; format=flowed; charset=iso-8859-1"), + "text/plain; format=flowed; charset=iso-8859-1", + ); + assert.equal( + contentTypeWithUtf8Charset("text/plain; format=flowed; ChArSeT = utf-16"), + "text/plain; format=flowed; ChArSeT = utf-16", + ); + assert.equal(contentTypeWithUtf8Charset('text/plain; note="one;two"'), 'text/plain; note="one;two"; charset=utf-8'); +}); + describe("agent files self-API", async () => { let server: Server; let base: string; let built: BuiltApp; let mineId: string; let theirsId: string; + let encodedId: string; + let binaryId: string; const capFor = (actorId: string) => mintCapabilityToken( @@ -41,6 +60,8 @@ describe("agent files self-API", async () => { base = `http://localhost:${(server.address() as AddressInfo).port}`; mineId = fileArtifactId("mine", "out", 0); theirsId = fileArtifactId("theirs", "out", 0); + encodedId = fileArtifactId("encoded", "out", 0); + binaryId = fileArtifactId("binary", "out", 0); await built.files.put({ id: mineId, ownerScopeId: scopeId("personal", "U1"), @@ -61,6 +82,26 @@ describe("agent files self-API", async () => { data: Buffer.from("theirs"), direction: "out", }); + await built.files.put({ + id: encodedId, + ownerScopeId: scopeId("personal", "U1"), + createdBy: "U1", + name: "encoded.json", + path: `artifacts/${encodedId}/encoded.json`, + mimetype: "application/json; charset=iso-8859-1", + data: Buffer.from("{}"), + direction: "out", + }); + await built.files.put({ + id: binaryId, + ownerScopeId: scopeId("personal", "U1"), + createdBy: "U1", + name: "binary.png", + path: `artifacts/${binaryId}/binary.png`, + mimetype: "image/png", + data: Buffer.from([0x89, 0x50, 0x4e, 0x47]), + direction: "out", + }); }); after(async () => { @@ -75,10 +116,7 @@ describe("agent files self-API", async () => { const res = await get("/v1/files", await capFor("U1")); assert.equal(res.status, 200); const page = (await res.json()) as { owned: Array<{ id: string }>; shared: Array<{ id: string }> }; - assert.deepEqual( - page.owned.map((file) => file.id), - [mineId], - ); + assert.deepEqual(page.owned.map((file) => file.id).sort(), [mineId, encodedId, binaryId].sort()); assert.deepEqual(page.shared, []); }); @@ -86,4 +124,16 @@ describe("agent files self-API", async () => { const res = await get(`/v1/files/${theirsId}/content`, await capFor("U1")); assert.equal(res.status, 404); }); + + it("adds UTF-8 to textual file content without changing explicit charsets or binary types", async () => { + const token = await capFor("U1"); + const text = await get(`/v1/files/${mineId}/content`, token); + const encoded = await get(`/v1/files/${encodedId}/content`, token); + const binary = await get(`/v1/files/${binaryId}/content`, token); + + assert.equal(text.headers.get("content-type"), "text/plain; charset=utf-8"); + assert.equal(encoded.headers.get("content-type"), "application/json; charset=iso-8859-1"); + assert.equal(binary.headers.get("content-type"), "image/png"); + await Promise.all([text.arrayBuffer(), encoded.arrayBuffer(), binary.arrayBuffer()]); + }); }); diff --git a/test/background-exec-broker.test.ts b/test/background-exec-broker.test.ts index 74b707fdf..6ad27156a 100644 --- a/test/background-exec-broker.test.ts +++ b/test/background-exec-broker.test.ts @@ -20,6 +20,7 @@ function fakeSandbox(opts?: { seedOutput?: string }) { const starts: Array<{ command: string; opts?: StartProcessOptions }> = []; const signals: Array<{ id: string; signal: string }> = []; const writes: Array<{ id: string; data: string }> = []; + const readErrors = new Map(); let n = 0; const sandbox: ProcessSandbox = { @@ -53,6 +54,11 @@ function fakeSandbox(opts?: { seedOutput?: string }) { return { processId }; }, async readProcess(_h, id, opts) { + if (readErrors.has(id)) { + const error = readErrors.get(id); + readErrors.delete(id); + throw error; + } if (missing.has(id)) throw new Error(`no such process session: ${id}`); const p = procs.get(id); if (!p) throw new Error(`no such process session: ${id}`); @@ -104,6 +110,7 @@ function fakeSandbox(opts?: { seedOutput?: string }) { } }, vanish: (id: string) => missing.add(id), + failNextRead: (id: string, error: unknown) => readErrors.set(id, error), }; } @@ -366,6 +373,17 @@ test("liveness probe: a row whose backend process is gone is deleted and a fresh assert.equal(rows[0]!.processId, second.processId); }); +test("liveness probe: a transient backend error preserves the running process and registry row", async () => { + const { broker, registry, starts, failNextRead } = build(); + const first = await broker.start(handle, "long-build"); + const error = new Error("backend unavailable"); + failNextRead(first.processId, error); + + await assert.rejects(broker.start(handle, "long-build"), error); + assert.equal(starts.length, 1); + assert.equal((await registry.get(first.processId))?.status, "running"); +}); + test("TTL clamp: a requested lifetime above the max is clamped (mirrors PR C's ceiling clamp)", async () => { const ttlMaxMs = 60 * 60_000; const { broker, registry } = build({ ttlMs: 30 * 60_000, ttlMaxMs }); diff --git a/test/command-policy.test.ts b/test/command-policy.test.ts index b6451ccd5..787d83dad 100644 --- a/test/command-policy.test.ts +++ b/test/command-policy.test.ts @@ -446,3 +446,27 @@ test("parseCommandPolicy rejects regexes with catastrophic repetition", () => { assert.ok("error" in parsed, pattern); } }); +test("a heredoc fed to a non-interpreter command (cat, gh) is data, not gated", () => { + const policy = defaultOrgPolicy(); + const prBody = [ + `gh pr create --title x --body "$(cat <<'EOF'`, + "Extract SQL payloads safely; previously DROP TABLE users in payloads broke parsing.", + "EOF", + ')"', + ].join("\n"); + assert.equal(evaluateCommand(prBody, policy).decision, "allow"); + const piped = ["cat <<'EOF' | gh pr create --body-file -", "fixes DROP TABLE handling", "EOF"].join("\n"); + assert.equal(evaluateCommand(piped, policy).decision, "allow"); +}); + +test("a heredoc fed to a SQL client or interpreter stays gated", () => { + const policy = defaultOrgPolicy(); + const sql = ["psql mydb < { + const policy = defaultOrgPolicy(); + const sneaky = ["cat < boolean): LeaderLease { return { @@ -22,6 +24,7 @@ function fakeLease(isLeader: () => boolean): LeaderLease { function harness( reply: string | ((req: TurnRequest) => Promise) = "CRON-OUTPUT-XYZ", directory?: DirectoryStore, + maxFiresPerTick?: number, ) { const crons = createCronStore(); const deliveries = createDeliveryStore(); @@ -42,6 +45,7 @@ function harness( identity, run, ...(directory ? { directory } : {}), + ...(maxFiresPerTick !== undefined ? { maxFiresPerTick } : {}), }); return { crons, deliveries, calls, scheduler, identity }; } @@ -291,7 +295,13 @@ test("a recurring teammate-DM cron without current recipient consent is withheld }, }); await scheduler.runNow(cron.id); - assert.equal((await deliveries.pending("principal")).length, 0); + const pending = await deliveries.pending("principal"); + assert.equal(pending.length, 1); + assert.equal(pending[0]?.destination.target, "U1"); + assert.match(pending[0]?.text ?? "", /consent.*skipped/i); + const stored = await crons.get(cron.id); + assert.equal(stored?.fireLog?.[0]?.status, "refused"); + assert.match(stored?.fireLog?.[0]?.note ?? "", /consent/); }); test("a teammate-DM cron created in a channel delivers its real output (§10 parity gate)", async () => { @@ -481,6 +491,191 @@ test("the default (no-op) lease ticks exactly as before — memory-mode behavior assert.equal(calls.length, 1, "without a lease, the tick fires due crons as before"); }); +test("a failing interval cron does not starve later due crons across ticks", async () => { + const { crons, calls, scheduler } = harness(async (req) => { + if (req.text.includes("fail first")) throw new Error("cron failed"); + return { status: "ok", reply: "OUT" }; + }); + const failing = await crons.create({ + schedule: { everyMs: 1000, firstFireAt: 1 }, + action: "fail first", + owner: "U1", + createdBy: "U1", + ownerScopeId: scopeId("personal", "U1"), + }); + const succeeding = await crons.create({ + schedule: { everyMs: 1000, firstFireAt: 1 }, + action: "succeed second", + owner: "U1", + createdBy: "U1", + ownerScopeId: scopeId("personal", "U1"), + }); + + await scheduler.tick(2000); + await scheduler.tick(3500); + + assert.deepEqual( + calls.map((call) => call.idempotencyKey), + [`cron:${failing.id}:1`, `cron:${succeeding.id}:1`, `cron:${failing.id}:1`, `cron:${succeeding.id}:3000`], + ); +}); + +test("a capped batch of persistent failures rotates — later due crons still get their turn", async () => { + const { crons, calls, scheduler } = harness( + async (req) => { + if (req.text.includes("always fails")) throw new Error("cron failed"); + return { status: "ok", reply: "OUT" }; + }, + undefined, + 2, + ); + for (let i = 0; i < 3; i++) { + await crons.create({ + schedule: { everyMs: 60_000, firstFireAt: 1 }, + action: `always fails ${i}`, + owner: "U1", + createdBy: "U1", + ownerScopeId: scopeId("personal", "U1"), + }); + } + const healthy = await crons.create({ + schedule: { everyMs: 60_000, firstFireAt: 1 }, + action: "healthy last in line", + owner: "U1", + createdBy: "U1", + ownerScopeId: scopeId("personal", "U1"), + }); + + await scheduler.tick(2000); + await scheduler.tick(3000); + + assert.ok( + calls.some((call) => call.idempotencyKey === `cron:${healthy.id}:1`), + "the healthy cron behind a full batch of failures fires within two ticks", + ); +}); + +test("capped rotation is driven by durable attempt order, not tick arrival times", async () => { + const { crons, calls, scheduler } = harness("OUT", undefined, 2); + const created: Cron[] = []; + for (let i = 0; i < 4; i++) { + created.push( + await crons.create({ + schedule: { everyMs: 600_000, firstFireAt: 1 }, + action: `cron ${i}`, + owner: "U1", + createdBy: "U1", + ownerScopeId: scopeId("personal", "U1"), + }), + ); + } + + await scheduler.tick(2000); + await scheduler.tick(4000); + + for (const cron of created) { + assert.ok( + calls.some((call) => call.idempotencyKey === `cron:${cron.id}:1`), + "every due cron fires within ceil(due/cap) ticks even when tick timestamps skip buckets", + ); + } +}); + +test("a cron whose attempt marker cannot persist is held back and cannot starve the rest", async () => { + const backing = createMemoryMap(); + const crons = createCronStore(backing); + const deliveries = createDeliveryStore(); + const calls: TurnRequest[] = []; + const run = async (req: TurnRequest): Promise => { + calls.push(req); + return { status: "ok", reply: "OUT" }; + }; + const created: Cron[] = []; + for (let i = 0; i < 5; i++) { + created.push( + await crons.create({ + schedule: { everyMs: 600_000, firstFireAt: 1 }, + action: `cron ${i}`, + owner: "U1", + createdBy: "U1", + ownerScopeId: scopeId("personal", "U1"), + }), + ); + } + const broken = created[0]!; + const flaky: typeof crons = { + ...crons, + async markAttempted(id, at) { + if (id === broken.id) throw new Error("attempt marker write failed"); + return crons.markAttempted(id, at); + }, + }; + const scheduler = createScheduler({ + crons: flaky, + deliveries, + idempotency: createIdempotencyStore(), + identity: createIdentityService(), + run, + maxFiresPerTick: 2, + }); + + await scheduler.tick(2000); + await scheduler.tick(4000); + + assert.ok( + !calls.some((call) => call.idempotencyKey === `cron:${broken.id}:1`), + "a capped batch never fires a cron whose attempt failed to persist", + ); + for (const cron of created.slice(1)) { + assert.ok( + calls.some((call) => call.idempotencyKey === `cron:${cron.id}:1`), + "the crons behind the broken marker still fire within two capped ticks", + ); + } +}); + +test("capped rotation survives a scheduler restart mid-cycle", async () => { + const backing = createMemoryMap(); + const crons = createCronStore(backing); + const deliveries = createDeliveryStore(); + const calls: TurnRequest[] = []; + const run = async (req: TurnRequest): Promise => { + calls.push(req); + return { status: "ok", reply: "OUT" }; + }; + const mk = () => + createScheduler({ + crons, + deliveries, + idempotency: createIdempotencyStore(), + identity: createIdentityService(), + run, + maxFiresPerTick: 2, + }); + const created: Cron[] = []; + for (let i = 0; i < 4; i++) { + created.push( + await crons.create({ + schedule: { everyMs: 600_000, firstFireAt: 1 }, + action: `cron ${i}`, + owner: "U1", + createdBy: "U1", + ownerScopeId: scopeId("personal", "U1"), + }), + ); + } + + await mk().tick(2000); + await mk().tick(2000); + + for (const cron of created) { + assert.ok( + calls.some((call) => call.idempotencyKey === `cron:${cron.id}:1`), + "a fresh scheduler instance resumes the rotation from durable state instead of restarting it", + ); + } +}); + test("runNow fires even when the current schedule slot already did (manual re-run)", async () => { const { crons, calls, scheduler } = harness(); const cron = await crons.create({ @@ -724,7 +919,7 @@ test("a scopeFloor cron whose only members are non-internal fails closed", async assert.equal((await crons.get(cron.id))?.enabled, false); }); -test("a failing background tick is logged, not swallowed", async (t) => { +test("a failing cron fire is logged, not swallowed", async (t) => { t.mock.timers.enable({ apis: ["setInterval"] }); const logged: string[] = []; t.mock.method(console, "error", (...args: unknown[]) => { @@ -749,13 +944,13 @@ test("a failing background tick is logged, not swallowed", async (t) => { }); scheduler.start(1000); t.mock.timers.tick(1000); - for (let i = 0; i < 50 && !logged.some((l) => l.includes("[scheduler] tick failed")); i++) { + for (let i = 0; i < 50 && !logged.some((l) => l.includes("[scheduler] fire failed")); i++) { await new Promise((r) => setImmediate(r)); } scheduler.stop(); assert.ok( - logged.some((l) => l.includes("[scheduler] tick failed") && l.includes("boom")), - "the tick error must reach the log", + logged.some((l) => l.includes("[scheduler] fire failed") && l.includes("boom")), + "the fire error must reach the log", ); const after = await crons.get(cron.id); assert.equal(after?.fireLog?.length, 1); diff --git a/test/deploy-warming-page.test.ts b/test/deploy-warming-page.test.ts new file mode 100644 index 000000000..e6e493403 --- /dev/null +++ b/test/deploy-warming-page.test.ts @@ -0,0 +1,117 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { createServer as createHttpServer } from "node:http"; +import type { AddressInfo } from "node:net"; +import { createInsecureTestServer } from "../src/api/server.ts"; +import type { App } from "../src/api/app.ts"; + +function appWith(endpoint: Record): App { + return { reachDeployment: async () => ({ status: "ok", endpoint }) } as unknown as App; +} + +test("/d/ proxy serves the warming page to a browser navigation when the deployment hangs", async () => { + const held: import("node:net").Socket[] = []; + const upstream = createHttpServer((req) => { + held.push(req.socket); + }); + upstream.listen(0); + const upstreamPort = (upstream.address() as AddressInfo).port; + + const server = createInsecureTestServer(appWith({ host: "127.0.0.1", port: upstreamPort }), { + deployDialTimeoutMs: 200, + }); + server.listen(0); + const base = `http://localhost:${(server.address() as AddressInfo).port}`; + try { + const res = await fetch(`${base}/d/some-id/`, { + headers: { accept: "text/html,application/xhtml+xml", "sec-fetch-dest": "document" }, + }); + assert.equal(res.status, 503); + assert.match(String(res.headers.get("content-type")), /text\/html/); + assert.equal(res.headers.get("retry-after"), "2"); + const body = await res.text(); + assert.match(body, /starting up/i); + assert.match(body, /location\.reload/); + } finally { + for (const s of held) s.destroy(); + await new Promise((r) => server.close(() => r())); + await new Promise((r) => upstream.close(() => r())); + } +}); + +test("/d/ proxy serves the warming page to a browser navigation when the deployment refuses connections", async () => { + const upstream = createHttpServer(() => {}); + upstream.listen(0); + const upstreamPort = (upstream.address() as AddressInfo).port; + await new Promise((r) => upstream.close(() => r())); + + const server = createInsecureTestServer(appWith({ host: "127.0.0.1", port: upstreamPort })); + server.listen(0); + const base = `http://localhost:${(server.address() as AddressInfo).port}`; + try { + const res = await fetch(`${base}/d/some-id/`, { headers: { accept: "text/html" } }); + assert.equal(res.status, 503); + assert.match(await res.text(), /starting up/i); + } finally { + await new Promise((r) => server.close(() => r())); + } +}); + +test("/d/ proxy keeps JSON gateway errors for non-document requests", async () => { + const held: import("node:net").Socket[] = []; + const upstream = createHttpServer((req) => { + held.push(req.socket); + }); + upstream.listen(0); + const upstreamPort = (upstream.address() as AddressInfo).port; + + const server = createInsecureTestServer(appWith({ host: "127.0.0.1", port: upstreamPort }), { + deployDialTimeoutMs: 200, + }); + server.listen(0); + const base = `http://localhost:${(server.address() as AddressInfo).port}`; + try { + const apiRes = await fetch(`${base}/d/some-id/api/data`, { headers: { accept: "application/json" } }); + assert.equal(apiRes.status, 504); + assert.equal(((await apiRes.json()) as { error?: string }).error, "gateway_timeout"); + + const postRes = await fetch(`${base}/d/some-id/`, { + method: "POST", + headers: { accept: "text/html", "content-type": "text/plain", "content-length": "2" }, + body: "hi", + }); + assert.equal(postRes.status, 504); + assert.equal(((await postRes.json()) as { error?: string }).error, "gateway_timeout"); + } finally { + for (const s of held) s.destroy(); + await new Promise((r) => server.close(() => r())); + await new Promise((r) => upstream.close(() => r())); + } +}); + +test("a recently-healthy upstream keeps the full dial timeout for slow pages", async () => { + let slow = false; + const upstream = createHttpServer((req, res) => { + if (slow) setTimeout(() => res.end("slow-ok"), 300); + else res.end("fast-ok"); + }); + upstream.listen(0); + const upstreamPort = (upstream.address() as AddressInfo).port; + + const server = createInsecureTestServer(appWith({ host: "127.0.0.1", port: upstreamPort }), { + deployDialTimeoutMs: 1000, + }); + server.listen(0); + const base = `http://localhost:${(server.address() as AddressInfo).port}`; + try { + const first = await fetch(`${base}/d/some-id/`, { headers: { accept: "text/html" } }); + assert.equal(await first.text(), "fast-ok"); + slow = true; + const second = await fetch(`${base}/d/some-id/`, { headers: { accept: "text/html" } }); + assert.equal(second.status, 200); + assert.equal(await second.text(), "slow-ok"); + } finally { + await new Promise((r) => server.close(() => r())); + await new Promise((r) => upstream.close(() => r())); + } +}); diff --git a/test/deployment-layer-routes.test.ts b/test/deployment-layer-routes.test.ts index 08323464f..c3edbaebd 100644 --- a/test/deployment-layer-routes.test.ts +++ b/test/deployment-layer-routes.test.ts @@ -2,7 +2,7 @@ import "./support/auto-fake-sprites.ts"; import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AddressInfo } from "node:net"; @@ -18,8 +18,8 @@ import { DeploymentLayerPersistedError } from "../src/deployment/deployment-laye const SECRET = "layer-routes-secret".repeat(3); const PATH = "/v1/deployment-layer"; -function start(overrides: { deploymentLayerDir?: string } = {}, serverDeps: Record = {}) { - const built = buildApp(testConfig({ signingSecret: SECRET, ...overrides })); +function start(overrides: Parameters[0] = {}, serverDeps: Record = {}) { + const built = buildApp(testConfig({ signingSecret: SECRET, sandboxBackend: "local", ...overrides })); const server = createServer(built.app, { signingSecret: SECRET, deploymentLayer: built.deploymentLayerStore, @@ -109,6 +109,39 @@ test("a signed PUT lands under portal-identity enforcement (deploy-time sync is } }); +test("a primary or secondary Sprites backend rejects tool-bearing deployment layers before persisting them", async () => { + for (const config of [ + { sandboxBackend: "sprites" as const }, + { sandboxBackend: "local" as const, sandboxSecondaryBackend: "sprites" as const }, + ]) { + const srv = start(config); + try { + const put = await fetch(`${srv.base}${PATH}`, { method: "PUT", headers: signed("PUT", bundle), body: bundle }); + assert.equal(put.status, 400); + const body = (await put.json()) as { error: string; message: string }; + assert.equal(body.error, "invalid_deployment_layer"); + assert.match(body.message, /does not support deployment-layer tools/); + assert.equal(await srv.deploymentLayerStore.get(), null); + } finally { + await srv.close(); + } + } +}); + +test("a secondary Sprites backend rejects a baked tool-bearing layer at boot", () => { + const dir = mkdtempSync(join(tmpdir(), "layer-routes-secondary-sprites-")); + mkdirSync(join(dir, "tools", "acme"), { recursive: true }); + writeFileSync(join(dir, "tools", "acme", "tool.json"), JSON.stringify({ id: "acme", advertise: "acme CLI" })); + try { + assert.throws( + () => start({ deploymentLayerDir: dir, sandboxBackend: "local", sandboxSecondaryBackend: "sprites" }), + /does not support deployment-layer tools/, + ); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + test("a correctly signed PUT replaces the layer and a signed GET reads it back", async () => { const srv = start(); try { diff --git a/test/deployment-layer-store.test.ts b/test/deployment-layer-store.test.ts index 3e969555a..ff461dbf5 100644 --- a/test/deployment-layer-store.test.ts +++ b/test/deployment-layer-store.test.ts @@ -117,6 +117,41 @@ test("invalid descriptors never replace the durable current layer", async () => assert.equal((await store.get())?.version, 1); }); +test("a store bundle validator rejects new and persisted unsupported tools before applying them", async () => { + const backing = createMemoryMap(); + const org = scopeId("org", "default-org"); + const writer = createDeploymentLayerStore({ + backing, + runtime: emptyDeploymentLayer(), + skills: createSkillStore({ signingSecret: "layer-test" }), + scopeId: org, + }); + await writer.put({ contract: 1, tools: [tool("acme")], skills: [] }, "old"); + + const runtime = emptyDeploymentLayer(); + const guarded = createDeploymentLayerStore({ + backing, + runtime, + skills: createSkillStore({ signingSecret: "layer-test" }), + scopeId: org, + validateBundle: (bundle) => { + if (bundle.tools.length) throw new Error("tools unsupported"); + }, + }); + const priorError = console.error; + console.error = (): void => {}; + try { + await guarded.hydrate(); + } finally { + console.error = priorError; + } + assert.deepEqual(runtime.tools, []); + await assert.rejects( + guarded.put({ contract: 1, tools: [tool("acme")], skills: [] }, "new"), + DeploymentLayerValidationError, + ); +}); + test("a legacy published skill with an unsafe name is quarantined without blocking layer replacement", async () => { const skillBacking = createMemoryMap(); const org = scopeId("org", "default-org"); diff --git a/test/device-flow-persist.test.ts b/test/device-flow-persist.test.ts index 06d619676..f7bf2bc8c 100644 --- a/test/device-flow-persist.test.ts +++ b/test/device-flow-persist.test.ts @@ -44,13 +44,15 @@ function sprites() { } const rw = (scope: string) => [{ scopeId: scope, mountPath: "", mode: "rw" as const }]; -function acmecliBrokeredLayer(): string { +function acmecliBrokeredLayer(binary?: string, approvals?: Array<{ pattern: string; reason?: string }>): string { const dir = mkdtempSync(join(tmpdir(), "dfp-layer-")); mkdirSync(join(dir, "tools/acmecli"), { recursive: true }); writeFileSync( join(dir, "tools/acmecli/tool.json"), JSON.stringify({ id: "acmecli", + ...(binary ? { install: { binary } } : {}), + ...(approvals ? { approvals } : {}), auth: { check: "acmecli me", reauth: "acmecli login --use-device-code", @@ -73,6 +75,187 @@ test("deviceFlowCredOwner: the person on their own personal box, the scope on a assert.equal(deviceFlowCredOwner(scopeId("personal", "U2"), "U1"), scopeId("personal", "U2")); }); +test("personal ephemeral-only credentials run only through credential_exec and are redacted", async () => { + let assumes = 0; + const sentinels = { + access: "AKIA_CREDENTIAL_EXEC_SENTINEL", + secret: "credential_exec_secret_sentinel", + token: "credential_exec_session_sentinel", + }; + const built = buildApp( + testConfig({ + dataDir: mkdtempSync(join(tmpdir(), "dfp-credential-exec-")), + signingSecret: "device-flow-test-secret", + deploymentLayerDir: acmecliBrokeredLayer("env"), + }), + { + credentialBrokers: { + acmecli: createAwsRoleBroker({ + roleArn: "arn:aws:iam::123456789012:role/acmecli-broker", + region: "us-west-2", + sessionActions: ["execute-api:Invoke"], + assumeRole: async () => { + assumes++; + return { + Credentials: { + AccessKeyId: sentinels.access, + SecretAccessKey: sentinels.secret, + SessionToken: sentinels.token, + Expiration: new Date(Date.now() + 3_600_000), + }, + }; + }, + }), + }, + }, + ); + const personal = scopeId("personal", actor.externalId); + const conversation = { + kind: "dm" as const, + threadRef: "dm:credential-exec", + audience: [actor], + }; + await built.deviceFlowCutover.set(personal, "acmecli", "ephemeral_only", "security@example.com"); + const ambient = await built.app.turn({ + surface: "slack", + actor, + conversation, + text: "!run printf '%s' \"${AWS_ACCESS_KEY_ID-unset}\"", + }); + assert.equal(ambient.reply, "unset"); + assert.equal(assumes, 0); + const direct = await built.app.turn({ surface: "slack", actor, conversation, text: "!run env" }); + assert.match(`${direct.reason ?? ""} ${direct.reply ?? ""}`, /credential_exec/); + const brokered = await built.app.turn({ + surface: "slack", + actor, + conversation, + text: "!credential acmecli []", + }); + assert.equal(assumes, 1); + assert.match(brokered.reply ?? "", //); + assert.match(brokered.reply ?? "", //); + assert.match(brokered.reply ?? "", //); + for (const value of Object.values(sentinels)) assert.doesNotMatch(brokered.reply ?? "", new RegExp(value)); + const durable = JSON.stringify(await built.sessions.getEntries(brokered.sessionId!)); + for (const value of Object.values(sentinels)) assert.doesNotMatch(durable, new RegExp(value)); + assert.equal( + ff.names().some((name) => name.includes("credential-exec")), + false, + ); +}); + +test("credential_exec honors deployment approval rules before vending credentials", async () => { + let assumes = 0; + const built = buildApp( + testConfig({ + dataDir: mkdtempSync(join(tmpdir(), "dfp-credexec-approval-")), + signingSecret: "device-flow-test-secret", + deploymentLayerDir: acmecliBrokeredLayer("env", [ + { pattern: "\\benv\\b\\s+tool\\b", reason: "mutating subcommand" }, + ]), + }), + { + credentialBrokers: { + acmecli: createAwsRoleBroker({ + roleArn: "arn:aws:iam::123456789012:role/acmecli-broker", + region: "us-west-2", + sessionActions: ["execute-api:Invoke"], + assumeRole: async () => { + assumes++; + return { + Credentials: { + AccessKeyId: "AKIA_APPROVAL_GATE", + SecretAccessKey: "approval_gate_secret_value", + SessionToken: "approval_gate_session_token", + Expiration: new Date(Date.now() + 3_600_000), + }, + }; + }, + }), + }, + }, + ); + const personal = scopeId("personal", actor.externalId); + const conversation = { kind: "dm" as const, threadRef: "dm:credexec-approval", audience: [actor] }; + await built.deviceFlowCutover.set(personal, "acmecli", "ephemeral_only", "security@example.com"); + + const gated = await built.app.turn({ + surface: "slack", + actor, + conversation, + text: '!credential acmecli ["tool","delete"]', + }); + assert.equal(gated.status, "pending_approval"); + assert.equal(assumes, 0, "no AssumeRole call happens for a blocked command"); + const pending = gated.pendingApprovals![0]!; + assert.match(pending.reason, /mutating subcommand/); + + const approved = await built.app.turn({ + surface: "slack", + actor, + conversation, + text: '!credential acmecli ["tool","delete"]', + approval: { requestId: pending.requestId, approved: true }, + }); + assert.equal(approved.status, "ok", approved.reason); + assert.equal(assumes, 1, "approval unblocks exactly one vended invocation"); + + const unrelated = await built.app.turn({ + surface: "slack", + actor, + conversation: { ...conversation, threadRef: "dm:credexec-approval-3" }, + text: '!credential acmecli ["me"]', + }); + assert.equal(unrelated.status, "ok", "subcommands without approval rules run without a grant"); + assert.equal(assumes, 1, "the broker's per-actor credential cache is reused within its TTL"); +}); + +test("a scope allow rule cannot override the ephemeral_only direct-execution deny", async () => { + const built = buildApp( + testConfig({ + dataDir: mkdtempSync(join(tmpdir(), "dfp-credexec-scope-allow-")), + signingSecret: "device-flow-test-secret", + deploymentLayerDir: acmecliBrokeredLayer("env"), + }), + { + credentialBrokers: { + acmecli: createAwsRoleBroker({ + roleArn: "arn:aws:iam::123456789012:role/acmecli-broker", + region: "us-west-2", + sessionActions: ["execute-api:Invoke"], + assumeRole: async () => ({ + Credentials: { + AccessKeyId: "AKIA_SCOPE_ALLOW", + SecretAccessKey: "scope_allow_secret_value", + SessionToken: "scope_allow_session_token", + Expiration: new Date(Date.now() + 3_600_000), + }, + }), + }), + }, + }, + ); + const personal = scopeId("personal", actor.externalId); + built.config.setCommandPolicy(personal, { + mode: "denylist", + rules: [{ pattern: "\\benv\\b", decision: "allow" }], + }); + const conversation = { kind: "dm" as const, threadRef: "dm:credexec-scope-allow", audience: [actor] }; + await built.deviceFlowCutover.set(personal, "acmecli", "ephemeral_only", "security@example.com"); + + const direct = await built.app.turn({ surface: "slack", actor, conversation, text: "!run env" }); + assert.match(`${direct.reason ?? ""} ${direct.reply ?? ""}`, /credential_exec/); + + const sanctioned = await built.app.turn({ + surface: "slack", + actor, + conversation: { ...conversation, threadRef: "dm:credexec-scope-allow-2" }, + text: "!credential acmecli []", + }); + assert.equal(sanctioned.status, "ok", sanctioned.reason); +}); + test("capture saves changed login bundles per service and fingerprint-skips unchanged ones", async () => { const sb = sprites(); const k = kc(); @@ -483,8 +666,8 @@ test("shared ACMECLI cutover isolates brokered STS without shrinking the existin }); assert.equal( brokeredAcmecli.reply, - "AKIA_BOB_GENERAL|AKIA_BOB", - "only ACMECLI gets brokered identity; adjacent AWS work keeps Bob's general authority", + "AKIA_BOB_GENERAL|AKIA_BOB_GENERAL", + "prefer-ephemeral direct execution retains the owner's legacy fallback without broker vending", ); const unpoisoned = await built.app.turn({ @@ -500,8 +683,9 @@ test("shared ACMECLI cutover isolates brokered STS without shrinking the existin assert.ok( ownerAudit.some((event) => event.action === "keychain.materialize" && event.resource.includes("owner-auth box")), ); - assert.ok( - ownerAudit.some((event) => event.action === "credential.materialize" && event.resource.includes("acmecli")), + assert.equal( + ownerAudit.some((event) => event.action === "credential.materialize"), + false, ); const scoped = await built.app.turn({ @@ -545,8 +729,8 @@ test("shared ACMECLI cutover isolates brokered STS without shrinking the existin assert.equal(aliceAcmecli.status, "ok", aliceAcmecli.reason); assert.equal( aliceAcmecli.reply, - "AKIA_ALICE|unset|absent", - "ambient shared-room ACMECLI keeps the acting user's identity without Bob's keychain", + "|unset|absent", + "direct execution has no brokered identity and no access to Bob's keychain", ); assert.equal( ff.names().some((n) => n.includes("scratch")), @@ -584,8 +768,10 @@ test("shared ACMECLI cutover isolates brokered STS without shrinking the existin "ephemeral-only removes already-materialized legacy files without deleting the stored record", ); const acmecliUsage = await built.credentialUsage.list({ slug: "acmecli" }); - assert.ok(acmecliUsage.some((row) => row.status === "ephemeral_vended" && row.principalId === "BOB")); - assert.ok(acmecliUsage.some((row) => row.status === "ephemeral_vended" && row.principalId === "ALICE")); + assert.equal( + acmecliUsage.some((row) => row.status === "ephemeral_vended"), + false, + ); const legacyUsage = await built.credentialUsage.list({ slug: "keychain:acmecli" }); assert.ok( legacyUsage.some((row) => row.status === "legacy_retained"), @@ -742,6 +928,15 @@ test("prefer-isolated keeps legacy ACMECLI when STS vending fails; isolated-only text: "!run cat ~/.acmecli/session.json", }); assert.equal(fallback.reply, "legacy_ok"); + await assert.rejects( + built.app.turn({ + surface: "slack", + actor, + conversation: { ...conversation, threadRef: "ch:C-acmecli-fallback:prefer-broker" }, + text: "!credential acmecli []", + }), + /could not vend credentials/, + ); await built.deviceFlowCutover.set(room, "acmecli", "ephemeral_only", "security@example.com"); const closed = await built.app.turn({ @@ -764,6 +959,15 @@ test("prefer-isolated keeps legacy ACMECLI when STS vending fails; isolated-only "absent", "isolated-only never restores an owner's ambient ACMECLI after broker failure", ); + await assert.rejects( + built.app.turn({ + surface: "slack", + actor, + conversation: { ...conversation, threadRef: "ch:C-acmecli-fallback:only-broker" }, + text: "!credential acmecli []", + }), + /could not vend credentials/, + ); const usage = await built.credentialUsage.list({ slug: "acmecli" }); assert.ok(usage.some((row) => row.status === "legacy_fallback")); assert.ok(usage.some((row) => row.status === "ephemeral_failed_closed")); diff --git a/test/egress-authz.test.ts b/test/egress-authz.test.ts index 2715a42fa..0f851b604 100644 --- a/test/egress-authz.test.ts +++ b/test/egress-authz.test.ts @@ -7,7 +7,7 @@ import { createRelayAuditSink, tokenFromRequest, hostFromAuthority, - isLinkLocalOrMetadataIp, + isBlockedDestinationIp, type EgressAuthzDeps, } from "../src/egress-authz-main.ts"; import { verifySignature } from "../src/auth/source-auth.ts"; @@ -111,26 +111,31 @@ test("hostFromAuthority strips ports and IPv6 brackets", () => { assert.equal(hostFromAuthority(""), null); }); -test("isLinkLocalOrMetadataIp covers IMDS, 169.254/16, fe80::/10, v4-mapped, AWS v6 IMDS", () => { - assert.equal(isLinkLocalOrMetadataIp("169.254.169.254"), true); - assert.equal(isLinkLocalOrMetadataIp("169.254.1.1"), true); - assert.equal(isLinkLocalOrMetadataIp("::ffff:169.254.169.254"), true); - assert.equal(isLinkLocalOrMetadataIp("fe80::1"), true); - assert.equal(isLinkLocalOrMetadataIp("febf::1"), true); - assert.equal(isLinkLocalOrMetadataIp("fd00:ec2::254"), true); - assert.equal(isLinkLocalOrMetadataIp("[fd00:ec2::254]"), true); - assert.equal(isLinkLocalOrMetadataIp("8.8.8.8"), false); - assert.equal(isLinkLocalOrMetadataIp("2606:4700::1111"), false); - assert.equal(isLinkLocalOrMetadataIp("fec0::1"), false); +test("isBlockedDestinationIp covers IMDS, 169.254/16, fe80::/10, v4-mapped, AWS v6 IMDS", () => { + assert.equal(isBlockedDestinationIp("169.254.169.254"), true); + assert.equal(isBlockedDestinationIp("169.254.1.1"), true); + assert.equal(isBlockedDestinationIp("::ffff:169.254.169.254"), true); + assert.equal(isBlockedDestinationIp("fe80::1"), true); + assert.equal(isBlockedDestinationIp("febf::1"), true); + assert.equal(isBlockedDestinationIp("fd00:ec2::254"), true); + assert.equal(isBlockedDestinationIp("[fd00:ec2::254]"), true); + assert.equal(isBlockedDestinationIp("8.8.8.8"), false); + assert.equal(isBlockedDestinationIp("2606:4700::1111"), false); + assert.equal(isBlockedDestinationIp("fec0::1"), false); }); -test("isLinkLocalOrMetadataIp catches non-canonical spellings of blocked addresses", () => { - assert.equal(isLinkLocalOrMetadataIp("::ffff:a9fe:a9fe"), true); - assert.equal(isLinkLocalOrMetadataIp("[::ffff:a9fe:a9fe]"), true); - assert.equal(isLinkLocalOrMetadataIp("fd00:ec2:0:0:0:0:0:254"), true); - assert.equal(isLinkLocalOrMetadataIp("fd00:0ec2::0254"), true); - assert.equal(isLinkLocalOrMetadataIp("FE80::1"), true); - assert.equal(isLinkLocalOrMetadataIp("fe80::1%eth0"), true); +test("isBlockedDestinationIp catches non-canonical spellings of blocked addresses", () => { + assert.equal(isBlockedDestinationIp("::ffff:a9fe:a9fe"), true); + assert.equal(isBlockedDestinationIp("[::ffff:a9fe:a9fe]"), true); + assert.equal(isBlockedDestinationIp("fd00:ec2:0:0:0:0:0:254"), true); + assert.equal(isBlockedDestinationIp("fd00:0ec2::0254"), true); + assert.equal(isBlockedDestinationIp("FE80::1"), true); + assert.equal(isBlockedDestinationIp("fe80::1%eth0"), true); + assert.equal(isBlockedDestinationIp("0.0.0.0"), true); + assert.equal(isBlockedDestinationIp("0.0.0.7"), true); + assert.equal(isBlockedDestinationIp("::"), true); + assert.equal(isBlockedDestinationIp("::ffff:0:0"), true); + assert.equal(isBlockedDestinationIp("[::]"), true); }); test("valid token, open policy => 200 + audited as ok with scope/principal", async () => { @@ -263,6 +268,23 @@ test("a name RESOLVING to link-local / denied IP is denied (rebind guard)", asyn } }); +// A forward proxy dials from ITS namespace, so an allowed `CONNECT 127.0.0.1:` would bridge +// any caller to the services colocated with the proxy — this decision service, Envoy's admin. +test("loopback destinations are denied even under an open policy, by literal and by resolved IP", async () => { + const { server, records } = boot({ lookup: async () => ["127.0.0.1"] }); + const port = await listen(server); + try { + const open = await egressToken({ allowedHosts: [] }); + for (const authority of ["127.0.0.1:9901", "127.0.0.1:48081", "[::1]:9901", "127.1.2.3:80"]) { + assert.equal(await check(port, authority, open), 403, authority); + } + assert.equal(await check(port, "rebind.example.com:9901", open), 403); + assert.equal(records.at(-1)?.verdict, "denied"); + } finally { + await close(server); + } +}); + test("a name resolving to a policy-denied IP is denied", async () => { const { server } = boot({ lookup: async () => ["10.9.9.9"] }); const port = await listen(server); diff --git a/test/file-artifact-store.test.ts b/test/file-artifact-store.test.ts index 8de0fc162..a2b01b7ff 100644 --- a/test/file-artifact-store.test.ts +++ b/test/file-artifact-store.test.ts @@ -1,6 +1,6 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdtemp, readdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -76,6 +76,21 @@ test("DurableByteStore (local-fs) round-trips binary intact across a fresh store } }); +test("DurableByteStore (local-fs) accepts concurrent identical writes", async () => { + const dir = await mkdtemp(join(tmpdir(), "docstore-")); + try { + const bytes = createLocalDurableByteStore(dir); + const big = Buffer.concat([PNG, Buffer.alloc(4 * 1024 * 1024, 7)]); + const results = await Promise.all(Array.from({ length: 8 }, () => bytes.put(big))); + for (const r of results) assert.equal(r.blobKey, results[0]!.blobKey); + assert.deepEqual(await drain(bytes as never, results[0]!.blobKey), big); + const leftovers = (await readdir(join(dir, "files"))).filter((name) => name.endsWith(".part")); + assert.deepEqual(leftovers, [], "no orphaned partial files survive the race"); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); + test("DurableByteStore enforces maxBytes mid-stream", async () => { const bytes = createMemoryDurableByteStore(); await assert.rejects(bytes.put(Buffer.alloc(1000), { maxBytes: 10 }), ByteSourceTooLargeError); diff --git a/test/monitor-broker.test.ts b/test/monitor-broker.test.ts index c6eec7d67..1a27949c2 100644 --- a/test/monitor-broker.test.ts +++ b/test/monitor-broker.test.ts @@ -1,13 +1,20 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { createMonitorBroker } from "../src/monitors/monitor-broker.ts"; +import { createMonitorBroker, readBackgroundOutputTail } from "../src/monitors/monitor-broker.ts"; +import type { BackgroundWatchArmedResult, BackgroundWatchResult } from "../src/monitors/monitor-broker.ts"; import { createMonitorStore } from "../src/monitors/monitor-store.ts"; import { createMemoryProcessRegistry } from "../src/processes/process-registry.ts"; import { scopeId } from "../src/types.ts"; const SCOPE = "personal:U1"; +type ReadOutputTail = Parameters[0]["readOutputTail"]; -async function harness() { +function armed(r: BackgroundWatchResult): BackgroundWatchArmedResult { + if ("completed" in r) assert.fail("expected watch to arm a monitor"); + return r; +} + +async function harness(readOutputTail: ReadOutputTail = async () => ({ outputTail: "" })) { const store = createMonitorStore(); const registry = createMemoryProcessRegistry(); const rec = await registry.register({ @@ -20,6 +27,7 @@ async function harness() { const broker = createMonitorBroker({ store, registry, + readOutputTail, scopeId: SCOPE, owner: "U1", ownerScopeId: scopeId("personal", "U1"), @@ -32,7 +40,7 @@ async function harness() { test("watch arms a monitor inheriting the turn's owner, scope, thread, and destination", async () => { const { broker, store, rec } = await harness(); - const r = await broker.watch("p-1", { instructions: "summarize failures", pattern: "FAIL", sinceCursor: 7 }); + const r = armed(await broker.watch("p-1", { instructions: "summarize failures", pattern: "FAIL", sinceCursor: 7 })); assert.equal(r.reattached, false); assert.equal(r.expiresAt, rec.expiresAt + 1000); const m = await store.get(r.monitorId); @@ -48,8 +56,8 @@ test("watch arms a monitor inheriting the turn's owner, scope, thread, and desti test("watching the same job in the same thread twice reattaches instead of double-arming", async () => { const { broker, store } = await harness(); - const first = await broker.watch("p-1"); - const second = await broker.watch("p-1"); + const first = armed(await broker.watch("p-1")); + const second = armed(await broker.watch("p-1")); assert.equal(second.reattached, true); assert.equal(second.monitorId, first.monitorId); assert.equal((await store.list()).length, 1); @@ -57,9 +65,9 @@ test("watching the same job in the same thread twice reattaches instead of doubl test("a repeat watch re-arms with the new settings instead of silently keeping stale ones", async () => { const { broker, store } = await harness(); - const first = await broker.watch("p-1", { pattern: "FAIL", instructions: "old", sinceCursor: 5 }); + const first = armed(await broker.watch("p-1", { pattern: "FAIL", instructions: "old", sinceCursor: 5 })); await store.advance(first.monitorId, { cursor: 9, tail: "partial" }); - const second = await broker.watch("p-1", { pattern: "ERROR", instructions: "new", sinceCursor: 20 }); + const second = armed(await broker.watch("p-1", { pattern: "ERROR", instructions: "new", sinceCursor: 20 })); assert.equal(second.reattached, true); const m = await store.get(first.monitorId); assert.equal(m?.pattern, "ERROR"); @@ -74,7 +82,7 @@ test("a repeat watch re-arms with the new settings instead of silently keeping s assert.equal(after?.cursor, 30); }); -test("watch refuses a job from another scope, an unknown job, and an exited job", async () => { +test("watch refuses a job from another scope and an unknown job", async () => { const { broker, registry } = await harness(); await registry.register({ processId: "p-other", @@ -85,8 +93,59 @@ test("watch refuses a job from another scope, an unknown job, and an exited job" }); await assert.rejects(() => broker.watch("p-other"), /no such background job/); await assert.rejects(() => broker.watch("p-missing"), /no such background job/); +}); + +test("watch on an exited job with output returns its final tail without arming a monitor", async () => { + const calls: Array<{ processId: string; maxBytes: number }> = []; + const { broker, registry, store } = await harness(async (processId, maxBytes) => { + calls.push({ processId, maxBytes }); + return { outputTail: "last lines\nfinished\n", cursor: 8192, exitCode: 0 }; + }); + await registry.markStatus("p-1", "exited"); + + const r = await broker.watch("p-1"); + + assert.deepEqual(r, { + processId: "p-1", + completed: true, + registryStatus: "exited", + exitCode: 0, + outputTail: "last lines\nfinished\n", + cursor: 8192, + }); + assert.deepEqual(calls, [{ processId: "p-1", maxBytes: 4096 }]); + assert.equal((await store.list()).length, 0); +}); + +test("watch on an exited job with no output returns success without arming a monitor", async () => { + const { broker, registry, store } = await harness(async () => ({ outputTail: "", cursor: 0 })); await registry.markStatus("p-1", "exited"); - await assert.rejects(() => broker.watch("p-1"), /already exited/); + + const r = await broker.watch("p-1"); + + assert.deepEqual(r, { + processId: "p-1", + completed: true, + registryStatus: "exited", + outputTail: "", + cursor: 0, + }); + assert.equal((await store.list()).length, 0); +}); + +test("readBackgroundOutputTail scans output and keeps the final bytes", async () => { + const output = "abcdefghij"; + + const r = await readBackgroundOutputTail(5, async (cursor, maxBytes) => { + const next = Math.min(output.length, cursor + maxBytes); + return { + chunks: output.slice(cursor, next), + cursor: next, + ...(next === output.length ? { exitCode: 0 } : {}), + }; + }); + + assert.deepEqual(r, { outputTail: "fghij", cursor: 10, exitCode: 0 }); }); test("watch rejects an invalid regex pattern up front", async () => { @@ -99,7 +158,7 @@ test("watch rejects an invalid regex pattern up front", async () => { test("unwatch removes own monitors only", async () => { const { broker, store } = await harness(); - const r = await broker.watch("p-1"); + const r = armed(await broker.watch("p-1")); const other = await store.create({ owner: "U2", createdBy: "U2", diff --git a/test/pi-tools.test.ts b/test/pi-tools.test.ts index 5e99d0f80..42e9159c7 100644 --- a/test/pi-tools.test.ts +++ b/test/pi-tools.test.ts @@ -1104,6 +1104,134 @@ test("tool entries carry the call id + faithful model-facing result (WAL replay assert.match(result("call-exec").result, /\[exit 0\]/); }); +test("memory remember accepts facts as a single string and records coercion", async () => { + const emitted: Emitted[] = []; + const remembered: string[][] = []; + const ref: ToolContextRef = { + current: { + ...fakeToolContext(), + async memoryRemember(facts) { + remembered.push(facts); + return facts.length; + }, + }, + emit: (e) => { + emitted.push(e as Emitted); + }, + scopeLabel: "personal:U1", + }; + const memory = createPiTools(ref).find((tool) => tool.name === "memory"); + + await call(memory, { action: "remember", facts: "Owns billing." }); + + assert.deepEqual(remembered, [["Owns billing."]]); + const result = emitted.find((e) => e.type === "tool_result" && e.payload.tool === "memory")!.payload; + assert.equal(result.coercedFrom, "facts"); + assert.equal(result.added, 1); +}); + +test("memory remember falls back to content lines with bullets", async () => { + const emitted: Emitted[] = []; + const remembered: string[][] = []; + const ref: ToolContextRef = { + current: { + ...fakeToolContext(), + async memoryRemember(facts) { + remembered.push(facts); + return facts.length; + }, + }, + emit: (e) => { + emitted.push(e as Emitted); + }, + scopeLabel: "personal:U1", + }; + const memory = createPiTools(ref).find((tool) => tool.name === "memory"); + + await call(memory, { action: "remember", facts: [], content: "\n- Owns billing.\n- Likes short updates.\n\n" }); + + assert.deepEqual(remembered, [["Owns billing.", "Likes short updates."]]); + const result = emitted.find((e) => e.type === "tool_result" && e.payload.tool === "memory")!.payload; + assert.equal(result.coercedFrom, "content"); + assert.equal(result.added, 2); +}); + +test("memory remember falls back to query when facts and content are empty", async () => { + const emitted: Emitted[] = []; + const remembered: string[][] = []; + const ref: ToolContextRef = { + current: { + ...fakeToolContext(), + async memoryRemember(facts) { + remembered.push(facts); + return facts.length; + }, + }, + emit: (e) => { + emitted.push(e as Emitted); + }, + scopeLabel: "personal:U1", + }; + const memory = createPiTools(ref).find((tool) => tool.name === "memory"); + + await call(memory, { action: "remember", facts: [], content: " ", query: "Prefers email summaries." }); + + assert.deepEqual(remembered, [["Prefers email summaries."]]); + const result = emitted.find((e) => e.type === "tool_result" && e.payload.tool === "memory")!.payload; + assert.equal(result.coercedFrom, "query"); + assert.equal(result.added, 1); +}); + +test("memory remember keeps normal facts arrays unchanged", async () => { + const emitted: Emitted[] = []; + const remembered: string[][] = []; + const ref: ToolContextRef = { + current: { + ...fakeToolContext(), + async memoryRemember(facts) { + remembered.push(facts); + return facts.length; + }, + }, + emit: (e) => { + emitted.push(e as Emitted); + }, + scopeLabel: "personal:U1", + }; + const memory = createPiTools(ref).find((tool) => tool.name === "memory"); + + await call(memory, { action: "remember", facts: ["Owns billing.", "Likes short updates."] }); + + assert.deepEqual(remembered, [["Owns billing.", "Likes short updates."]]); + const result = emitted.find((e) => e.type === "tool_result" && e.payload.tool === "memory")!.payload; + assert.equal(result.coercedFrom, undefined); + assert.equal(result.added, 2); +}); + +test("memory remember all-empty error names the supplied fields", async () => { + const emitted: Emitted[] = []; + const ref: ToolContextRef = { + current: fakeToolContext(), + emit: (e) => { + emitted.push(e as Emitted); + }, + scopeLabel: "personal:U1", + }; + const memory = createPiTools(ref).find((tool) => tool.name === "memory"); + + const ret = (await call(memory, { action: "remember", facts: [], content: "", query: "" })) as { + content: Array<{ text: string }>; + }; + + assert.equal( + ret.content[0]!.text, + "[error] memory remember requires `facts` (a non-empty list). Received: facts, content, query (use facts instead).", + ); + const result = emitted.find((e) => e.type === "tool_result" && e.payload.tool === "memory")!.payload; + assert.equal(result.isError, true); + assert.equal(result.error, "facts required"); +}); + test("not-found read and denied command record isError + the faithful error text", async () => { const emitted: Emitted[] = []; const denyTC: ToolContext = { @@ -1178,6 +1306,72 @@ test("execute forwards the agent's timeout_seconds into tc.execute; omitting it assert.equal(sink.lastExecOpts, undefined); }); +test("credential_exec is turn-scoped, typed, and forwards only service plus literal argv", async () => { + const calls: unknown[] = []; + const tc: ToolContext = { + ...fakeToolContext(), + credentialExecServices: [{ service: "acme", binary: "acmecli" }], + async credentialExec(service, args, opts) { + calls.push({ service, args, opts }); + return { stdout: "authenticated", stderr: "", code: 0, timedOut: false }; + }, + }; + const absent = createPiTools({ current: fakeToolContext() }); + assert.equal( + absent.some((tool) => tool.name === "credential_exec"), + false, + ); + const tools = createPiTools( + { current: tc }, + { credentialExecServices: tc.credentialExecServices, execTimeoutCeilingMs: 10_000 }, + ); + const tool = tools.find((candidate) => candidate.name === "credential_exec")!; + assert.match(tool.description, /acme \(acmecli\)/); + assert.match(tool.description, /Shell operators and pipelines are not supported/); + const args = ["; env", "$(env)", "a|b", "> out", "two words"]; + const result = await call(tool, { service: "acme", args, timeout_seconds: 7 }); + assert.deepEqual(calls, [{ service: "acme", args, opts: { timeoutSeconds: 7 } }]); + assert.match((result as { content: Array<{ text: string }> }).content[0]!.text, /authenticated/); +}); + +test("credential_exec surfaces NeedsApproval and CommandDenied like execute", async () => { + const gated: ToolContext = { + ...fakeToolContext(), + credentialExecServices: [{ service: "acme", binary: "acmecli" }], + async credentialExec() { + throw new NeedsApproval("'acmecli' 'tool'", "mutating subcommand", "approval", "tool", "\\bacmecli\\s+tool\\b"); + }, + }; + const ref = { current: gated, pendingApprovals: [] as NonNullable }; + const tool = createPiTools(ref, { credentialExecServices: gated.credentialExecServices }).find( + (candidate) => candidate.name === "credential_exec", + )!; + const blocked = (await call(tool, { service: "acme", args: ["tool"] })) as { + content: Array<{ text: string }>; + terminate?: boolean; + }; + assert.match(blocked.content[0]!.text, /needs human approval/); + assert.equal(blocked.terminate, true); + assert.equal(ref.pendingApprovals.length, 1); + assert.equal(ref.pendingApprovals[0]!.approvalKey, "\\bacmecli\\s+tool\\b"); + assert.equal((ref as { pausedOnApproval?: boolean }).pausedOnApproval, true); + + const denied: ToolContext = { + ...fakeToolContext(), + credentialExecServices: [{ service: "acme", binary: "acmecli" }], + async credentialExec() { + throw new CommandDenied("'acmecli'", "must be run with credential_exec"); + }, + }; + const deniedTool = createPiTools({ current: denied }, { credentialExecServices: denied.credentialExecServices }).find( + (candidate) => candidate.name === "credential_exec", + )!; + const deniedResult = (await call(deniedTool, { service: "acme", args: [] })) as { + content: Array<{ text: string }>; + }; + assert.match(deniedResult.content[0]!.text, /denied by policy/); +}); + test('execute scope:"owner" routes only when the owner-auth surface is enabled', async () => { const sink: { lastExecOpts?: Parameters[1] } = {}; const execute = createPiTools({ current: fakeToolContext(sink) }, { ownerAuthExec: true })[0]!; @@ -1302,6 +1496,39 @@ test("background dispatches each action and emits tool_call/tool_result", async assert.equal(bg.length, 10); }); +test("background watch reports an already exited job as a successful tail result", async () => { + const textOf = (r: unknown): string => (r as { content: Array<{ text: string }> }).content[0]?.text ?? ""; + const emitted: Emitted[] = []; + const ref: ToolContextRef = { + current: { + ...fakeToolContext(), + async backgroundWatch(processId) { + return { + processId, + completed: true, + registryStatus: "exited", + exitCode: 0, + outputTail: "final line\n", + cursor: 42, + }; + }, + }, + emit: (e) => { + emitted.push(e as Emitted); + }, + scopeLabel: "personal:U1", + }; + const background = createPiTools(ref).find((t) => t.name === "background"); + + const watched = textOf(await call(background, { action: "watch", process_id: "bg-1" })); + + assert.match(watched, /job already exited \(code 0\) — no watch armed; here is the tail of its output:/); + assert.match(watched, /final line/); + const result = emitted.find((e) => e.type === "tool_result")!.payload; + assert.equal(result.completed, true); + assert.equal(result.error, undefined); +}); + test("background per-action validation returns a crisp [error] instead of throwing", async () => { const textOf = (r: unknown): string => (r as { content: Array<{ text: string }> }).content[0]?.text ?? ""; const background = createPiTools({ current: fakeToolContext() }).find((t) => t.name === "background"); @@ -1574,7 +1801,7 @@ test("guidance conversation scope reads the effective SOUL; write requires conte }); test("guidance defaults to channel scope when a channel is available, and rewrites the channel order", async () => { - assert.match(textOut(await call(tool("guidance"), { action: "read" })), /no channel guidance set/); + assert.match(textOut(await call(tool("guidance"), { action: "read" })), /Ambient replies: default/); assert.match( textOut(await call(tool("guidance"), { action: "write", content: "reply piratey to tweets" })), /channel guidance updated/, @@ -1585,7 +1812,44 @@ test("guidance defaults to channel scope when a channel is available, and rewrit ); assert.match( textOut(await call(tool("guidance"), { action: "write" })), - /\[error\].*needs `content`.*and\/or `bots`/, + /\[error\].*needs `content`.*`bots`.*and\/or `ambientEnabled`/, + ); +}); + +test("guidance reads and writes channel ambient replies without changing omitted state", async () => { + let ambientEnabled: boolean | undefined; + const tc: ToolContext = { + ...fakeToolContext(), + async getStandingOrder() { + return { ok: true, orders: "keep watch", ...(ambientEnabled === undefined ? {} : { ambientEnabled }) }; + }, + async setStandingOrder(orders, _bots, nextAmbientEnabled) { + if (nextAmbientEnabled !== undefined) ambientEnabled = nextAmbientEnabled ?? undefined; + return { ok: true, orders, ...(ambientEnabled === undefined ? {} : { ambientEnabled }) }; + }, + }; + + assert.match(textOut(await call(tool("guidance", tc), { action: "write", ambientEnabled: true })), /updated/); + assert.match(textOut(await call(tool("guidance", tc), { action: "read" })), /Ambient replies: on/); + await call(tool("guidance", tc), { action: "write", content: "keep watching" }); + assert.match(textOut(await call(tool("guidance", tc), { action: "read" })), /Ambient replies: on/); + await call(tool("guidance", tc), { action: "write", ambientEnabled: false }); + assert.match(textOut(await call(tool("guidance", tc), { action: "read" })), /Ambient replies: off/); + await call(tool("guidance", tc), { action: "write", ambientEnabled: null }); + assert.match(textOut(await call(tool("guidance", tc), { action: "read" })), /Ambient replies: default/); +}); + +test("guidance rejects ambient replies at conversation scope", async () => { + assert.match( + textOut( + await call(tool("guidance"), { + action: "write", + scope: "conversation", + content: "Be terse.", + ambientEnabled: true, + }), + ), + /\[error\].*applies only to channel scope/, ); }); diff --git a/test/postgres-store.test.ts b/test/postgres-store.test.ts index 2a3657b11..cc5103bb1 100644 --- a/test/postgres-store.test.ts +++ b/test/postgres-store.test.ts @@ -287,7 +287,10 @@ test("pg deleteSession: hard-removes the session and its rows, leaves others", { const keptRow = (await s.listByParticipant("DEL1")).find((x) => x.id === keep.id); assert.ok(keptRow, "other sessions untouched"); assert.equal(keptRow!.hasEntries, false, "an entry-less session reads back hasEntries=false"); - assert.notEqual((await s.acquireLease(a.id)).lease, null, "lease row cleared (re-acquirable)"); + assert.equal((await s.acquireLease(a.id)).lease, null, "no lease is granted on a deleted session"); + const reborn = await s.getOrCreateByThread("del-t1", "dm", scope); + assert.notEqual(reborn.id, a.id, "the thread gets a fresh session"); + assert.notEqual((await s.acquireLease(reborn.id)).lease, null, "the fresh session is leasable"); }); test("pg scopeSessionSummaries: counts via aggregate, not per-transcript reads", { skip }, async () => { @@ -1180,3 +1183,104 @@ test("pg participant view: pin/color are per-participant, survive re-add, and cl assert.ok(!cleared.pinned); assert.equal(cleared.color ?? null, null, "null clears the color"); }); + +test( + "pg deleteSessionIfEmpty: an expired lease forfeits, and the stale holder cannot orphan entries", + { skip }, + async () => { + const s = createPostgresSessionStore(URL!, { leaseTtlMs: 40 }); + const scope = scopeId("personal", "USTALE"); + const sess = await s.getOrCreateByThread("web:USTALE:seed", "dm", scope); + const att = await s.acquireLease(sess.id); + assert.ok(att.lease); + await new Promise((r) => setTimeout(r, 60)); + assert.equal(await s.deleteSessionIfEmpty(sess.id), true, "an expired lease does not block the discard"); + assert.equal(await s.get(sess.id), null); + await assert.rejects( + s.append(att.lease!, { type: "user", payload: {}, scopeLabel: scope }), + /valid session lease/, + "the stale holder cannot append after the discard", + ); + const pg = (await import("pg")).default; + const raw = new pg.Pool({ connectionString: URL }); + const leftovers = await raw.query( + "SELECT (SELECT COUNT(*) FROM session_entries WHERE session_id = $1) AS e, (SELECT COUNT(*) FROM session_leases WHERE session_id = $1) AS l", + [sess.id], + ); + await raw.end(); + assert.equal(Number(leftovers.rows[0].e), 0, "no orphaned entries after the stale append is refused"); + assert.equal(Number(leftovers.rows[0].l), 0, "no orphaned lease row survives"); + }, +); + +test("pg deleteSession: racing a fresh lease acquisition never orphans entries", { skip }, async () => { + const s = createPostgresSessionStore(URL!); + const scope = scopeId("personal", "UHARD"); + for (let i = 0; i < 15; i++) { + const sess = await s.getOrCreateByThread(`web:UHARD:${i}`, "dm", scope); + const [attempt] = await Promise.all([s.acquireLease(sess.id), s.deleteSession(sess.id)]); + assert.equal(await s.get(sess.id), null, "the session is gone either way"); + if (attempt.lease) { + await assert.rejects( + s.append(attempt.lease, { type: "user", payload: {}, scopeLabel: scope }), + /valid session lease/, + "a lease granted before the delete cannot orphan entries", + ); + } + } + const pg = (await import("pg")).default; + const raw = new pg.Pool({ connectionString: URL }); + const leftovers = await raw.query( + "SELECT COUNT(*) AS n FROM session_leases l WHERE NOT EXISTS (SELECT 1 FROM sessions ss WHERE ss.id = l.session_id)", + ); + await raw.end(); + assert.equal(Number(leftovers.rows[0].n), 0, "no lease row survives its session"); +}); + +test("pg deleteSessionIfEmpty: racing a fresh lease acquisition never orphans the lease", { skip }, async () => { + const s = createPostgresSessionStore(URL!); + const scope = scopeId("personal", "URACE"); + for (let i = 0; i < 15; i++) { + const sess = await s.getOrCreateByThread(`web:URACE:${i}`, "dm", scope); + const [attempt, discarded] = await Promise.all([s.acquireLease(sess.id), s.deleteSessionIfEmpty(sess.id)]); + if (discarded) { + assert.equal(attempt.lease, null, "a discarded session never grants a lease"); + assert.equal(await s.get(sess.id), null); + } else { + assert.ok(attempt.lease, "when the discard is refused the lease was granted"); + assert.ok(await s.get(sess.id), "the session survives when the lease won"); + await s.releaseLease(attempt.lease!); + assert.equal(await s.deleteSessionIfEmpty(sess.id), true); + } + } +}); + +test("pg deleteSessionIfEmpty: a held lease or landed entries refuse the discard atomically", { skip }, async () => { + const s = createPostgresSessionStore(URL!); + const scope = scopeId("personal", "UDISC"); + const sess = await s.getOrCreateByThread("web:UDISC:seed", "dm", scope); + const att = await s.acquireLease(sess.id); + assert.ok(att.lease); + assert.equal(await s.deleteSessionIfEmpty(sess.id), false, "a held lease blocks the discard"); + await s.append(att.lease, { type: "user", payload: { text: "seed" }, scopeLabel: scope }); + await s.releaseLease(att.lease); + assert.equal(await s.deleteSessionIfEmpty(sess.id), false, "entries block the discard"); + assert.ok(await s.get(sess.id), "the refused discard leaves the session intact"); + + const empty = await s.getOrCreateByThread("web:UDISC:seed2", "dm", scope); + const att2 = await s.acquireLease(empty.id); + assert.ok(att2.lease); + await s.releaseLease(att2.lease); + assert.equal(await s.deleteSessionIfEmpty(empty.id), true, "a released empty session is discarded"); + assert.equal(await s.get(empty.id), null); + + const pg = (await import("pg")).default; + const raw = new pg.Pool({ connectionString: URL }); + const orphans = await raw.query( + "SELECT (SELECT COUNT(*) FROM session_entries WHERE session_id = $1) AS e, (SELECT COUNT(*) FROM session_leases WHERE session_id = $1) AS l", + [empty.id], + ); + await raw.end(); + assert.equal(Number(orphans.rows[0].e), 0, "no orphaned entries survive the discard"); + assert.equal(Number(orphans.rows[0].l), 0, "no orphaned lease survives the discard"); +}); diff --git a/test/process-run.test.ts b/test/process-run.test.ts index 653c5fc89..2b0a14b8f 100644 --- a/test/process-run.test.ts +++ b/test/process-run.test.ts @@ -85,6 +85,31 @@ test("processRun threads runId + background into the turn and completes the run assert.equal(seen[1]?.background, true, "the worker-loop flag reaches the orchestrator"); }); +test("processRun rejects when a reaped attempt finishes after a retry claims the run", async () => { + const { runs } = createMemoryRunStore(); + let finish = (_: TurnResult) => {}; + const turnResult = new Promise((resolve) => { + finish = resolve; + }); + const orchestrator = fakeOrchestrator(() => turnResult); + + await runs.enqueue({ sessionId: "s1", request: turn }); + const first = await runs.claim("w1", -1); + const pending = processRun({ runs, orchestrator, leaseTtlMs: 5_000 }, first!); + + assert.deepEqual(await runs.reapExpired(), { requeued: 1, parked: 0 }); + const second = await runs.claim("w2", 5_000); + assert.equal(second?.attempts, 2); + + finish({ status: "ok", reply: "stale" }); + await assert.rejects(pending, /lost.*lease/i); + + const current = await runs.get(first!.id); + assert.equal(current?.status, "running"); + assert.equal(current?.leaseToken, second?.leaseToken); + assert.equal(current?.result, null); +}); + test("processRun upgrades legacy queued provenance before orchestration", async () => { const { runs } = createMemoryRunStore(); const legacy = { ...turn, origin: undefined, liveActor: true, triggerTs: "1" } as unknown as OrchestratorInput; diff --git a/test/session-store.test.ts b/test/session-store.test.ts index 42855ce56..38797944d 100644 --- a/test/session-store.test.ts +++ b/test/session-store.test.ts @@ -587,7 +587,10 @@ for (const [name, make] of backends) { true, "other sessions untouched", ); - assert.notEqual((await store.acquireLease(s.id)).lease, null, "lease row cleared (re-acquirable)"); + assert.equal((await store.acquireLease(s.id)).lease, null, "no lease is granted on a deleted session"); + const reborn = await store.getOrCreateByThread("t1", "dm", scope); + assert.notEqual(reborn.id, s.id, "the thread gets a fresh session"); + assert.notEqual((await store.acquireLease(reborn.id)).lease, null, "the fresh session is leasable"); }); test(`${name}: listByParticipant sets lastActivityAt to the most recent user message`, async () => { @@ -834,3 +837,34 @@ test("cronIdOf and sessionOrigin agree on which threadRefs are crons", () => { assert.equal(cronIdOf("cron:abc:slot"), "abc"); assert.equal(cronIdOf("dm:D1"), null); }); + +test("deleteSessionIfEmpty refuses while a lease is held and after entries land", async () => { + const nowRef = { v: 10_000_000_000 }; + const store = createMemorySessionStore({ now: () => nowRef.v, leaseTtlMs: 50 }); + const scope = scopeId("personal", "U1"); + const s = await store.getOrCreateByThread("web:U1:seed", "dm", scope); + const { lease } = await store.acquireLease(s.id); + assert.ok(lease); + assert.equal(await store.deleteSessionIfEmpty(s.id), false, "a held lease blocks the discard"); + await store.append(lease, { type: "user", payload: { text: "seed" }, scopeLabel: scope }); + await store.releaseLease(lease); + assert.equal(await store.deleteSessionIfEmpty(s.id), false, "entries block the discard"); + const empty = await store.getOrCreateByThread("web:U1:seed2", "dm", scope); + const second = await store.acquireLease(empty.id); + assert.ok(second.lease); + await store.releaseLease(second.lease); + assert.equal(await store.deleteSessionIfEmpty(empty.id), true, "a released empty session is discarded"); + assert.equal(await store.get(empty.id), null); + assert.equal((await store.acquireLease(empty.id)).lease, null, "no lease is granted on a discarded session"); + + const abandoned = await store.getOrCreateByThread("web:U1:seed3", "dm", scope); + const stale = await store.acquireLease(abandoned.id); + assert.ok(stale.lease); + nowRef.v += 60; + assert.equal(await store.deleteSessionIfEmpty(abandoned.id), true, "an expired lease does not block the discard"); + await assert.rejects( + store.append(stale.lease, { type: "user", payload: {}, scopeLabel: scope }), + /valid session lease/, + "the stale holder cannot append after the discard", + ); +}); diff --git a/test/surface-post-files.test.ts b/test/surface-post-files.test.ts index 67b950ada..325eafa0e 100644 --- a/test/surface-post-files.test.ts +++ b/test/surface-post-files.test.ts @@ -7,6 +7,7 @@ import { createMemoryFileArtifactStore } from "../src/files/file-artifact-store. import { createMemoryDurableByteStore } from "../src/files/durable-byte-store.ts"; import { scopeId } from "../src/types.ts"; import type { Sandbox, SandboxHandle } from "../src/sandbox/sandbox.ts"; +import { createMemoryChannelPolicyStore } from "../src/surface-cache/channel-policy-store.ts"; function fakeSandbox(files: Record, outboxListing: string[]): Sandbox { return { @@ -102,6 +103,38 @@ test("surface post rejects traversal before provisioning or staging any attachme assert.deepEqual(calls, { provision: 0, read: 0, put: 0, grant: 0 }); }); +test("surface standing orders preserve and reset the stored ambient reply policy", async () => { + const channelPolicy = createMemoryChannelPolicyStore(); + const tools = createSurfaceToolDeps({ + deps: { deliveries: {}, channelPolicy, auditLog: { record() {} } }, + input: { surfaceTools: true }, + actor: { id: "U1" }, + conversation: { kind: "channel", channelRef: "C1" }, + session: { id: "S1" }, + scopeId: "channel:C1", + defaultDestination: {}, + strictReadOnly: false, + blobTransfer: {}, + fileRegistration: {}, + provision: async () => handle, + postProvenance() { + return {}; + }, + spine: { surfaceOutboundCount: 0, crossConversationPosts: 0 }, + } as unknown as SurfaceToolsContext)!; + + await tools.setStandingOrder("watch", undefined, true); + assert.equal((await channelPolicy.get("C1"))?.ambientEnabled, true); + const enabledOrder = await tools.getStandingOrder(); + assert.equal(enabledOrder.ok && enabledOrder.ambientEnabled, true); + await tools.setStandingOrder("keep watching"); + assert.equal((await channelPolicy.get("C1"))?.ambientEnabled, true); + await tools.setStandingOrder("keep watching", undefined, null); + assert.equal((await channelPolicy.get("C1"))?.ambientEnabled, undefined); + const defaultOrder = await tools.getStandingOrder(); + assert.equal(defaultOrder.ok && defaultOrder.ambientEnabled, undefined); +}); + test("collectNamedOutbound: a missing/empty path is reported (so post can fail the WHOLE call)", async () => { const transfer = createMemoryBlobTransferStore(); const sandbox = fakeSandbox({ "outbox/there.png": bytes("X"), "outbox/blank.txt": bytes("") }, []); diff --git a/test/trigger-consent.test.ts b/test/trigger-consent.test.ts index dd799698f..7b5a3dbba 100644 --- a/test/trigger-consent.test.ts +++ b/test/trigger-consent.test.ts @@ -113,7 +113,8 @@ describe("runTrigger: recipient-consent gate", () => { recipientConsent: { recipientId: "U-alice", status: "pending" }, }); assert.equal( - (await deps.deliveries.pending("principal")).length, + (await deps.deliveries.pending("principal")).filter((delivery) => delivery.destination.target === "U-alice") + .length, 0, "nothing reaches a recipient who hasn't accepted", ); @@ -145,7 +146,11 @@ describe("runTrigger: recipient-consent gate", () => { destination: toAlice, recipientConsentRequired: true, }); - assert.equal((await deps.deliveries.pending("principal")).length, 0); + assert.equal( + (await deps.deliveries.pending("principal")).filter((delivery) => delivery.destination.target === "U-alice") + .length, + 0, + ); }); it("accepted consent for a prior recipient does not authorize a retargeted standing DM", async () => { @@ -160,7 +165,26 @@ describe("runTrigger: recipient-consent gate", () => { recipientConsent: { recipientId: "U-alice", status: "accepted" }, recipientConsentRequired: true, }); - assert.equal((await deps.deliveries.pending("principal")).length, 0); + assert.equal( + (await deps.deliveries.pending("principal")).filter((delivery) => delivery.destination.target === "U-bob").length, + 0, + ); + }); + + it("an unrelated turn refusal sends no consent notice (nothing was withheld for consent)", async () => { + const deps = triggerDeps(async () => ({ status: "refused", reason: "runtime not approved" })); + const out = await runTrigger(deps, { + owner: "U-carol", + ownerScopeId: scopeId("personal", "U-carol"), + input: "compose", + fireKey: "c-unrelated", + surface: "cron", + destination: toAlice, + recipientConsentRequired: true, + }); + assert.equal(out.status, "refused"); + assert.doesNotMatch(out.note ?? "", /consent/); + assert.equal((await deps.deliveries.pending("principal")).length, 0, "no misleading skip notice to anyone"); }); it("withholds a verbatim relay too (a declined recipient gets nothing)", async () => { @@ -177,7 +201,11 @@ describe("runTrigger: recipient-consent gate", () => { destination: toAlice, recipientConsent: { recipientId: "U-alice", status: "declined" }, }); - assert.equal((await deps.deliveries.pending("principal")).length, 0); + assert.equal( + (await deps.deliveries.pending("principal")).filter((delivery) => delivery.destination.target === "U-alice") + .length, + 0, + ); assert.match(out.note ?? "", /turned this delivery off/); }); }); diff --git a/test/visibility.test.ts b/test/visibility.test.ts index a4bcd6991..867cc7796 100644 --- a/test/visibility.test.ts +++ b/test/visibility.test.ts @@ -123,6 +123,7 @@ describe("runTrigger: fire-time visibility gate", () => { assert.equal(out.authzFailed, false, "not disabled — staleness must not kill a cron"); assert.equal((await deps.deliveries.pending("slack")).length, 0, "no post to a channel the owner can't see"); assert.match(out.note ?? "", /no longer visible/); + assert.equal(out.status, "refused", "a withheld delivery is not recorded as a success"); }); it("skips the run entirely (§10 leg a) when the actor lost the HOME scope — the exfil shape", async () => {