diff --git a/package.json b/package.json index 939ffe6..8c891fd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@openparachute/cloud", - "version": "0.0.8-rc.132", + "version": "0.0.8-rc.133", "private": true, "description": "Open Parachute PBC's Vault Cloud \u2014 one Durable Object per vault on Cloudflare, OAuth issuer + self-serve console (accounts + vault ownership).", "license": "AGPL-3.0", diff --git a/scripts/staging-sweep.ts b/scripts/staging-sweep.ts index 6bd2627..f57566c 100644 --- a/scripts/staging-sweep.ts +++ b/scripts/staging-sweep.ts @@ -16,12 +16,14 @@ * * WHAT IT ACTUALLY DELETES: only the D1 OWNERSHIP ROW (`vaults.name`) for * vaults matching the smoke's own throwaway prefixes, older than the cutoff. - * There is no vault-teardown verb anywhere in this codebase — no route drops - * a Durable Object's storage or purges its R2 attachments wholesale (see - * `workers/identity/src/account-api.ts`'s `DELETE /account/vaults/:name`, - * still a `501 not_implemented` stub, and `vault-do.ts`'s internal-verb - * table, which has no delete/wipe entry) — so the underlying DO SQLite - * storage + any R2 objects are left ORPHANED, not reclaimed. That's an + * A REAL teardown verb now exists (cloud#226): `vault-do.ts`'s + * `POST /api/internal/destroy` purges the DO storage + the whole + * `vault-/` R2 prefix, and `DELETE /account/vaults/:name` + * (account-api.ts) drives it plus the identity-side D1 sweep. THIS SCRIPT + * STILL DOES NEITHER — its trust root is the operator's own wrangler + * credential against D1, not a minted account bearer, so the underlying DO + * SQLite storage + any R2 objects are left ORPHANED, not reclaimed. Porting + * this lever onto the real delete door is separate work. That's an * acceptable trade for what this script is FOR: the debris problem is the D1 * `vaults` COUNT (what the console lists, what the usage/snapshot crons * enumerate, what the vault-count cap counts against, and what the nightly diff --git a/workers/identity/migrations/0024_account_delete_wiring.sql b/workers/identity/migrations/0024_account_delete_wiring.sql new file mode 100644 index 0000000..4bc40eb --- /dev/null +++ b/workers/identity/migrations/0024_account_delete_wiring.sql @@ -0,0 +1,55 @@ +-- Account deletion, part two (cloud#226 A-3/A-4): the two things migration +-- 0023's substrate could not know it needed until routes actually drove it. +-- 0023 added the columns; nothing wrote or looked anything up by them, so +-- neither of these gaps was reachable. Both are now. +-- +-- NOTE ON MIGRATION NUMBERING: 0023's header reserved 0024 for the +-- vault-delete train's PR-2a (`vaults.deleting_at` / `destroy_completed_at`). +-- That design is NOT being built — the vault delete shipped as an immediate, +-- confirm-guarded teardown with no staging columns (see +-- handleAccountVaultDelete's "NO UNDO WINDOW HERE"), so PR-2a has no columns +-- to add and the slot was free. Re-based at open time, per the same +-- assign-at-open discipline 0023 followed. +-- +-- 1. `idx_users_delete_undo_hash` — the undo endpoint +-- (`/account/undo-delete`) looks an account up BY THIS HASH, and it is the +-- one account surface that is UNAUTHENTICATED by construction: a +-- tombstoned account cannot present a bearer or a cookie (migration 0023's +-- chokepoints refuse both), so the mailed token is the only credential +-- there is. Without an index that lookup is a full table scan of `users` +-- on every request, which any unauthenticated caller can drive at will — +-- free amplification, and it gets worse exactly as the user table grows. +-- +-- UNIQUE, not merely an index, and that is the load-bearing half: the +-- lookup uses `.first()`, so two rows sharing a hash would resolve to an +-- ARBITRARY one of them — restoring the wrong account. A collision is not +-- reachable today (the token is 32 random bytes), but "not reachable +-- today" is exactly the class of assumption that rots silently; here the +-- database refuses instead. Same pattern and same reasoning as +-- `idx_users_drip_unsub_token` (migration 0008). +-- +-- SQLite treats NULLs as distinct in a UNIQUE index, so the overwhelming +-- majority of rows (every account that is not mid-deletion, all of which +-- carry NULL) are unconstrained — this cannot collide in normal operation. +CREATE UNIQUE INDEX idx_users_delete_undo_hash ON users (delete_undo_hash); + +-- 2. `users.delete_purge_attempts` — how many convergence-sweep passes have +-- tried and failed to finish this account's teardown (0/NULL = never +-- deferred; reset is unnecessary because a converged account's row is +-- deleted outright). +-- +-- Why it must exist: the sweep is deliberately conservative — a vault +-- whose DO destroy fails defers the WHOLE account rather than purging the +-- row that records whose the surviving vault is. That is the right call +-- per pass and the wrong one forever. Billing tears down on the FIRST +-- pass, so a permanently wedged vault leaves an account paying nothing, +-- invisible in the product, and still holding tenant data we told the user +-- in writing was "permanently deleted" 24 hours ago. Retrying silently is +-- how that becomes a year. +-- +-- The counter is what lets the sweep escalate: past a threshold it pages +-- the operator (ops_alerts) instead of quietly trying again. It is a +-- counter rather than a first-seen timestamp so the alert tracks ATTEMPTS +-- actually made — a cron that stops firing must not age an account into an +-- alert that claims work was tried. +ALTER TABLE users ADD COLUMN delete_purge_attempts INTEGER; diff --git a/workers/identity/src/account-api.ts b/workers/identity/src/account-api.ts index 3e485f4..911d390 100644 --- a/workers/identity/src/account-api.ts +++ b/workers/identity/src/account-api.ts @@ -17,6 +17,8 @@ * - list → `listVaultsForOwner` + the daily rollup's `vault_usage` rows; * - mint → the `vault-call.ts` first-party mint seam's `signAccessToken`, * ownership-gated (`userOwnsVault`), scope-validated to this one vault. + * - delete → the same first-party seam's destroy call, followed by the + * explicit identity-side D1 sweep (the vault worker is always first). * * THE TENANT-FACING VAULT TOKEN'S client_id — a deliberate divergence from the * plan's literal `client_id="parachute-console"`, and it is security-load- @@ -69,12 +71,13 @@ import { validateHandle, } from "./handles.ts"; import { type User, getUserById } from "./users.ts"; -import { pushVaultCap } from "./vault-call.ts"; +import { callVaultDestroy, pushVaultCap, readDestroyOutcome } from "./vault-call.ts"; import { VaultNameInvalidError, VaultNameTakenError, countVaultsForOwner, createVault, + deleteVaultD1Rows, listVaultsForOwner, userOwnsVault, } from "./vaults.ts"; @@ -97,7 +100,7 @@ function authError(status: number, error: string, description: string): Response /** * REST/business-error body (`{ error, message }`), matching the vault-call * response shape — used for resource-level failures (invalid name, taken, - * at-cap, not-owned, not-implemented). + * at-cap, not-owned, and teardown failures). */ function restError(status: number, error: string, message: string): Response { return jsonResponse({ error, message }, status, { "cache-control": "no-store" }); @@ -499,26 +502,129 @@ export async function handleAccountVaultCreate(db: D1Database, req: Request, dep ); } -// --- DELETE /account/vaults/ — not yet on the hosted door -------------- +// --- DELETE /account/vaults/ — destroy + identity cleanup --------------- /** - * Vault teardown. The cloud console has NO delete door today (verified: no - * vault-row delete / DO teardown anywhere in the identity worker), and inventing - * destructive machinery here — unreviewed, on a live tenant boundary — is out of - * scope. So this honestly answers 501: authenticated (an unauthenticated caller - * still gets 401, not a shape leak), but the operation isn't implemented. + * Delete an owned vault through the hosted door. The Bearer account gate and the + * retype-the-name guard run before the destructive call; ownership is checked + * against the account id in the bearer, so an unknown vault and another user's + * vault share the same neutral `not_owner` response. * - * When delete lands, wrap the real teardown here with the hub twin's confirm - * shape (`{ confirm: "" }` retype) + the `userOwnsVault` ownership gate, - * mirroring the hub's `handleDeleteVault`. Scope: `account::admin`. + * Order is load-bearing: the vault worker is the source of truth for whether + * tenant data actually disappeared, so its idempotent destroy call happens + * BEFORE the D1 sweep. A non-2xx or transport failure leaves every identity row + * intact for a retry. Once destroy succeeds, the D1 batch runs the same cascade + * the self-hosted hub twin runs — revoke the tokens naming this vault, rewrite + * the grants that name it, then drop the vault, usage, and snapshot rows; a D1 + * failure is reported separately because the storage side is already erased. + * + * Idempotency choice: after the first successful delete, the `vaults` row is + * gone. A second request therefore returns the same 403 `not_owner` shape as a + * first request for an unknown or unowned vault. This is intentional: it avoids + * an existence oracle while ensuring a re-delete never throws or returns 500. + * Scope: `account::admin`. + * + * NO UNDO WINDOW HERE — deliberately, and not to be confused with its sibling. + * The 24-hour undo window Aaron ratified belongs to ACCOUNT deletion (`DELETE + * /account`, the A-train: migration 0023's `users.deleted_at` tombstone, the + * `billing-teardown.ts` module, and the convergence sweep). That design can + * defer destruction because an account's teardown is a billing/auth state + * change first. A VAULT delete cannot: the only verb that erases vault content + * is the vault worker's `POST /api/internal/destroy` (cloud#226 PR-1), which is + * irreversible and immediate by construction — `deleteAll()` on the DO plus an + * R2 prefix purge. Staging a vault delete would need its own reversible + * substrate (the unmerged PR-2a `vaults.deleting_at` / `destroy_completed_at` + * columns plus a sweep); until that lands, the retype-the-name confirm IS the + * guard, exactly as the self-hosted hub twin (`handleDeleteVault`) has it. + * + * Convergence if the D1 sweep fails after a successful destroy: the response + * says so (`500 d1_cleanup_failed`) and the whole request is safe to repeat — + * destroy is idempotent on a warm instance and re-runs harmlessly on a cold + * one, and the D1 batch is written to be re-runnable. Until the retry lands, + * the stale `vaults` row presents as an empty vault, not as tenant data. */ export async function handleAccountVaultDelete(db: D1Database, req: Request, deps: OAuthDeps): Promise { const auth = await requireAccount(db, req, deps, "admin"); if (!auth.ok) return auth.response; - return restError( - 501, - "not_implemented", - "Vault deletion is not available on the hosted door yet. Contact hello@parachute.computer.", + + const name = vaultNameFromPath(req, ""); + if (!name) return restError(404, "not_found", "no such vault route"); + + const body = await readJsonBody(req); + if (body.confirm !== name) { + return restError(400, "confirm_mismatch", `deleting a vault requires the body {"confirm": "${name}"}`); + } + + // Ownership: false for BOTH "owned by someone else" and "doesn't exist" — + // one 403 for both, so deletion cannot become an existence oracle. This gate + // is intentionally before the first-party vault-worker call. + if (!(await userOwnsVault(db, auth.accountId, name))) { + return restError(403, "not_owner", `You do not own a vault named "${name}".`); + } + + let destroyResponse: Response; + try { + destroyResponse = await callVaultDestroy(db, deps, auth.accountId, name); + } catch (err) { + console.error( + `event=vault_destroy_failed vault=${name} error=${JSON.stringify(err instanceof Error ? err.message : String(err))}`, + ); + return restError( + 502, + "vault_destroy_failed", + "The vault storage teardown could not be reached. No identity rows were changed; retry the request.", + ); + } + // The one shared judgement (vault-call.ts readDestroyOutcome) — a 200 is not + // enough; the body must be the DO's real reply. See that function for why a + // false positive here is unrecoverable while a false negative costs a retry. + const outcome = await readDestroyOutcome(destroyResponse); + if (!outcome.ok) { + console.error(`event=vault_destroy_failed vault=${name} reason=${outcome.reason} detail=${JSON.stringify(outcome.detail)}`); + return restError( + 502, + "vault_destroy_failed", + "The vault storage teardown did not confirm success. No identity rows were changed; retry the request.", + ); + } + + let d1: Awaited>; + try { + d1 = await deleteVaultD1Rows(db, auth.accountId, name, deps.now?.() ?? new Date()); + } catch (err) { + console.error( + `event=vault_identity_cleanup_failed vault=${name} error=${JSON.stringify(err instanceof Error ? err.message : String(err))}`, + ); + return restError( + 500, + "d1_cleanup_failed", + "Vault storage was destroyed, but identity cleanup did not finish. Retry the delete request.", + ); + } + + // Ops audit line — the only durable record that a tenant's vault was torn + // down (the rows that would have shown it are exactly what just got deleted). + console.log( + `event=vault_deleted vault=${name} owner=${auth.accountId} r2_objects_deleted=${outcome.r2ObjectsDeleted} ` + + `vault_rows=${d1.vaultRowsDeleted} usage_rows=${d1.usageRowsDeleted} snapshot_rows=${d1.snapshotRowsDeleted} ` + + `tokens_revoked=${d1.tokensRevoked} grants_rewritten=${d1.grantsRewritten} grants_dropped=${d1.grantsDropped}`, + ); + + return jsonResponse( + { + destroyed: true, + r2_objects_deleted: outcome.r2ObjectsDeleted, + d1: { + vault_rows_deleted: d1.vaultRowsDeleted, + vault_usage_rows_deleted: d1.usageRowsDeleted, + vault_snapshot_rows_deleted: d1.snapshotRowsDeleted, + tokens_revoked: d1.tokensRevoked, + grants_rewritten: d1.grantsRewritten, + grants_dropped: d1.grantsDropped, + }, + }, + 200, + { "cache-control": "no-store" }, ); } diff --git a/workers/identity/src/account-delete.ts b/workers/identity/src/account-delete.ts new file mode 100644 index 0000000..6392827 --- /dev/null +++ b/workers/identity/src/account-delete.ts @@ -0,0 +1,697 @@ +/** + * Account deletion — the A-train's A-3 (the delete + undo routes) and A-4 (the + * convergence sweep), the wiring that finally makes A-1 and A-2 do something. + * + * Aaron's ruling, restated because every ordering decision below follows from + * it: deleting an account SEVERS AUTHENTICATION IMMEDIATELY but gives a + * 24-HOUR UNDO WINDOW before anything irreversible happens. Three moving parts, + * each already built and until now inert: + * + * - A-1 (migration 0023 + the read-time chokepoints): `users.deleted_at` is + * the tombstone. Every surface that can ACT on an account already refuses a + * tombstoned row — the bearer gate, the session JOIN, both password logins, + * magic-link request/consume, the drip, and the billing/usage/snapshot + * sweeps. THIS MODULE IS THE FIRST THING THAT WRITES THAT COLUMN. Because + * A-1 is already everywhere, the instant the tombstone lands the account + * stops working — that is what "severs auth immediately" means mechanically, + * and it is why the session/token wipe below is belt, not braces. + * - A-2 (billing-teardown.ts): `deferBilling` on request (reversible), + * `resumeBilling` on undo, `teardownBilling` at expiry (irreversible). + * - PR-1 + the vault-delete door (vault-call.ts `callVaultDestroy`, + * vaults.ts `deleteVaultD1Rows`): the only verb that actually erases tenant + * vault content. The sweep reuses it per owned vault rather than inventing + * a second teardown — which is why the vault-delete door and this land on + * one branch. + * + * THE ORDERING RULE, once, since it recurs: at REQUEST time, do the reversible + * thing first and the durable thing second, and compensate if the second fails + * — a hold with no tombstone is a bug we can unwind, a tombstone with no hold + * is a charge that posts against an account the user believes is closed. At + * SWEEP time, the reverse: billing before storage, storage before the user row, + * and every step idempotent, so a partial pass always converges on retry and + * never strands an un-cancelable subscription behind a deleted row that no + * longer names it. + * + * NO ORACLE, with ONE deliberate exception. Every undo token that cannot be + * used — unknown, malformed, already spent, belonging to a live account — + * gets the SAME neutral 400, so the endpoint never confirms whether an + * address has an account mid-deletion. The exception is a token that WAS + * valid and whose window has since closed: that gets an honest 410 + * `undo_window_expired`. Reaching it requires having held the real token, so + * it reveals nothing to anyone who didn't already have it, and "you're too + * late" is the only useful thing to tell the person who did. + * + * The undo endpoint is the ONE account surface that must work for a user whose + * auth was just severed, so it authenticates on the mailed token alone — never + * a session, never a bearer, both of which A-1 now refuses by construction. + */ +import type Stripe from "stripe"; +import { readJsonBody, requireAccount } from "./account-api.ts"; +import { billingConfig } from "./billing-config.ts"; +import type { BillingOverrides } from "./billing.ts"; +import { deferBilling, resumeBilling, teardownBilling } from "./billing-teardown.ts"; +import { randomBase64url, sha256Hex } from "./crypto.ts"; +import type { EmailSender } from "./email.ts"; +import type { Env } from "./env.ts"; +import { type OAuthDeps, jsonResponse } from "./oauth-shared.ts"; +import { deleteSessionsForUser } from "./sessions.ts"; +import { raiseOpsAlert } from "./ops-alerts.ts"; +import { makeStripe } from "./stripe-client.ts"; +import { type User, getUserById } from "./users.ts"; +import { callVaultDestroy, readDestroyOutcome } from "./vault-call.ts"; +import { deleteVaultD1Rows, listVaultsForOwner } from "./vaults.ts"; + +/** The ratified undo window: 24 hours from the delete REQUEST (`deleted_at`). */ +export const DELETE_UNDO_WINDOW_MS = 24 * 60 * 60 * 1000; + +/** + * Accounts converged per sweep pass. The sweep rides the hourly tick and each + * account costs a Stripe round-trip plus one DO wake per owned vault, so this + * caps the tail latency of a single cron invocation; the backlog drains at + * CAP/hour, which is far above any plausible delete rate. + */ +export const ACCOUNT_DELETE_SWEEP_CAP = 25; + +/** REST/business-error body (`{ error, message }`) — the `/account/*` shape. */ +function deleteError(status: number, error: string, message: string): Response { + return jsonResponse({ error, message }, status, { "cache-control": "no-store" }); +} + +/** + * The Stripe client for a billing operation, or a reason there isn't one. + * `overrides.stripe` is the test seam (billing.ts's `BillingOverrides`, the + * same one billing-lifecycle's tests use). An environment with no billing + * configured has no Stripe to talk to — which is FINE for an account carrying + * no Stripe ids and NOT fine for one that does, so the caller decides. + */ +function stripeFor(env: Env, overrides?: BillingOverrides): Stripe | null { + if (overrides?.stripe) return overrides.stripe; + const config = billingConfig(env); + return config ? makeStripe(config.secretKey) : null; +} + +/** Whether this account has anything in Stripe that a teardown must reach. */ +function hasBillingArtifacts(user: Pick): boolean { + return user.stripeCustomerId !== null || user.stripeSubscriptionId !== null; +} + +// --- A-3a: DELETE /account — request deletion, open the undo window ---------- + +/** + * Request account deletion. Scope: `account::admin` — the same gate every + * other `/account/*` mutation runs, and the account id comes from the bearer's + * `sub`, never a body field, so this can only ever delete the CALLER's account. + * + * Guarded by a retype confirm, exactly as the vault door is: the body must + * carry `{ "confirm": "" }`. The email is this surface's + * analogue of the vault name — the one string the account holder knows and a + * mis-aimed script does not. + * + * WHAT HAPPENS, in the order it happens and why: + * + * 1. `deferBilling` — the REVERSIBLE hold, FIRST. This is the "no new charge + * can post after a delete is requested" guarantee, and it must not depend + * on the D1 write that follows. A hard Stripe failure aborts the request + * with nothing written; `deferBilling` is idempotent, so a retry is safe. + * 2. The TOMBSTONE — `deleted_at` + `delete_undo_hash`, conditional on + * `deleted_at IS NULL` so two concurrent requests cannot both open a + * window (the second sees zero changed rows and is told so). If this + * write fails, the hold from step 1 is COMPENSATED with `resumeBilling` + * before answering — the one place this module unwinds itself, because a + * standing cancel-at-period-end on an account that was never deleted is + * silent money loss nobody would think to look for. + * 3. Auth severance — sessions dropped, live tokens revoked. Belt, not + * braces: A-1's chokepoints already refuse a tombstoned account at every + * read path, so the account is dead the instant step 2 commits whether or + * not this succeeds. It runs anyway so the rows don't sit there + * spendable-looking for a day, and a failure is logged, not fatal — + * failing the request here would be the worst outcome, since the delete + * HAS taken effect. + * 4. The notice email, carrying the undo link. Non-fatal for the same + * reason; `delete_notice_sent_at` is stamped only on a real send, so the + * column never claims an email that didn't go out. + * + * The undo token is returned in the response body as well as mailed. That is + * deliberate: the caller is the authenticated account admin who just asked for + * this, so the body tells them nothing they didn't just prove they're entitled + * to — and an API client (which may have no inbox at all) can offer undo + * without depending on email delivery. + * + * VAULT DATA IS UNTOUCHED HERE. Nothing tenant-visible is erased until the + * window closes and the sweep runs ({@link runAccountDeleteSweep}); during the + * window the data is intact but unreachable, because every read path already + * refuses a tombstoned owner. That is the whole point of the window. + */ +export async function handleAccountDelete( + env: Env, + req: Request, + deps: OAuthDeps, + sender?: EmailSender, + overrides?: BillingOverrides, +): Promise { + const auth = await requireAccount(env.DB, req, deps, "admin"); + if (!auth.ok) return auth.response; + const user = auth.user; + + const body = await readJsonBody(req); + const confirm = typeof body.confirm === "string" ? body.confirm.trim().toLowerCase() : null; + if (confirm === null || confirm !== user.email.trim().toLowerCase()) { + return deleteError( + 400, + "confirm_mismatch", + `deleting an account requires the body {"confirm": "${user.email}"}`, + ); + } + + // 1. The reversible hold, before anything durable. + const stripe = stripeFor(env, overrides); + if (user.stripeSubscriptionId !== null) { + if (!stripe) { + return deleteError( + 503, + "billing_not_configured", + "This account has a subscription but billing is not configured here, so the hold that stops new charges cannot be placed. Deletion was not started.", + ); + } + try { + await deferBilling(stripe, user.stripeSubscriptionId); + } catch (err) { + console.error( + `event=account_delete_hold_failed user=${user.id} error=${err instanceof Error ? err.message : String(err)}`, + ); + return deleteError( + 502, + "billing_hold_failed", + "Billing could not be paused, so deletion was not started. Nothing changed; retry the request.", + ); + } + } + + // 2. The tombstone — the durable fact that makes the account deleted. + const now = deps.now?.() ?? new Date(); + const nowIso = now.toISOString(); + const undoToken = randomBase64url(32); + const undoHash = await sha256Hex(undoToken); + let changed = 0; + try { + const res = await env.DB + .prepare("UPDATE users SET deleted_at = ?, delete_undo_hash = ? WHERE id = ? AND deleted_at IS NULL") + .bind(nowIso, undoHash, user.id) + .run(); + changed = res.meta.changes ?? 0; + } catch (err) { + console.error( + `event=account_delete_tombstone_failed user=${user.id} error=${err instanceof Error ? err.message : String(err)}`, + ); + // Compensate the step-1 hold — see the ordering note above. + if (stripe && user.stripeSubscriptionId !== null) { + try { + await resumeBilling(stripe, user.stripeSubscriptionId); + } catch (resumeErr) { + // BOTH halves failed: the subscription is now sitting at + // cancel_at_period_end = true for an account that was NOT deleted, and + // nothing in the system will ever revisit it — the user's next attempt + // starts from scratch, the sweep only looks at tombstoned rows, and + // there is no tombstone. The user finds out when their subscription + // lapses at the period boundary for no reason they can see. That is + // exactly the invisible-money-loss shape that earns the alert channel + // rather than a log line (ops-alerts.ts). + await raiseOpsAlert(env, sender, { + key: `billing-hold-stuck:${user.id}`, + subject: "billing hold stuck on a NON-deleted account", + text: [ + `A delete request placed a cancel_at_period_end hold, then failed to`, + `record the deletion AND failed to release the hold.`, + ``, + ` account: ${user.id}`, + ` subscription: ${user.stripeSubscriptionId}`, + ` release error: ${resumeErr instanceof Error ? resumeErr.message : String(resumeErr)}`, + ``, + `This account is NOT deleted and has no tombstone, so no sweep will`, + `ever revisit it. Its subscription will silently lapse at the period`, + `boundary unless someone clears cancel_at_period_end in Stripe.`, + ].join("\n"), + now: deps.now?.() ?? new Date(), + }); + } + } + return deleteError( + 500, + "delete_request_failed", + "The deletion could not be recorded. Nothing was deleted; retry the request.", + ); + } + if (changed === 0) { + // requireAccount already refuses a tombstoned account, so reaching here + // means a concurrent request won the race — report it rather than minting + // a second undo token that would silently orphan the first. + return deleteError(409, "already_deleting", "This account is already scheduled for deletion."); + } + + // 3. Sever the credentials that exist right now (belt — see the doc above). + try { + await deleteSessionsForUser(env.DB, user.id); + await env.DB + .prepare("UPDATE tokens SET revoked_at = ? WHERE user_id = ? AND revoked_at IS NULL") + .bind(nowIso, user.id) + .run(); + } catch (err) { + console.error( + `event=account_delete_severance_incomplete user=${user.id} error=${err instanceof Error ? err.message : String(err)}`, + ); + } + + // 4. The notice, carrying the way back. + const purgeAt = new Date(now.getTime() + DELETE_UNDO_WINDOW_MS); + const undoUrl = `${deps.issuer}/account/undo-delete?token=${encodeURIComponent(undoToken)}`; + let noticeSent = false; + if (sender) { + // TRANSPORT NOTE: this rides `sendOps` — plain text, no List-Unsubscribe + // headers, which is CORRECT for a transactional deletion notice (an + // unsubscribable "your account is closing" email would be a bug). What it + // does not yet have is a branded template like the magic-link and drip + // sends; that is presentation work, deliberately not blocking the door. + const result = await sender.sendOps({ + to: user.email, + subject: "Your Parachute account is scheduled for deletion", + text: [ + `We received a request to delete the Parachute account for ${user.email}.`, + ``, + `Your account is now closed and you have been signed out everywhere.`, + `Nothing has been erased yet.`, + ``, + `You have until ${purgeAt.toISOString()} (24 hours) to change your mind.`, + `After that, your vaults and their contents are permanently deleted and`, + `cannot be recovered.`, + ``, + `To restore your account, open this link before then:`, + `${undoUrl}`, + ``, + `If you did not request this, restore your account now and change your`, + `password.`, + ].join("\n"), + }); + if (result.ok) { + noticeSent = true; + try { + await env.DB.prepare("UPDATE users SET delete_notice_sent_at = ? WHERE id = ?").bind(nowIso, user.id).run(); + } catch { + // The email really went out; failing to stamp it is a bookkeeping + // miss, not a reason to tell the user their deletion failed. + } + } else { + console.error(`event=account_delete_notice_failed user=${user.id} error=${result.error}`); + } + } + + console.log(`event=account_delete_requested user=${user.id} purge_at=${purgeAt.toISOString()} notice_sent=${noticeSent}`); + return jsonResponse( + { + deleted: true, + deleted_at: nowIso, + undo_expires_at: purgeAt.toISOString(), + undo_token: undoToken, + undo_url: undoUrl, + notice_sent: noticeSent, + }, + 200, + { "cache-control": "no-store" }, + ); +} + +// --- A-3b: /account/undo-delete — the way back, inside the window ------------ + +/** The one neutral answer for every unusable token — see the module note. */ +function invalidUndoToken(): Response { + return deleteError(400, "invalid_undo_token", "That restore link is not valid."); +} + +/** + * Restore an account inside its undo window. Authenticated by the mailed token + * ALONE — no session, no bearer — because A-1 refuses both for a tombstoned + * account, so any other credential would make undo structurally impossible. + * + * Answers both GET (`?token=` — the link in the notice email, so a click + * restores) and POST (`{ "token": ... }` — API clients). A GET that mutates is + * a deliberate exception, on the same footing as the magic-link consume: the + * token is single-use and unguessable, and the mutation is the RESTORING + * direction, so a stray prefetch can only ever un-delete something the account + * holder asked to be able to un-delete. + * + * Order mirrors the request path in reverse, and again the reversible thing + * leads: billing is resumed BEFORE the tombstone clears, so an account that + * comes back always comes back with its subscription state already settled. + * A hard Stripe failure leaves the tombstone in place and the window still + * open — retryable, and the account stays honestly deleted meanwhile rather + * than half-restored. + * + * The one case that cannot be undone is reported, not hidden: if the billing + * period boundary passed during the window, Stripe already canceled for real + * and there is no un-cancel ({@link resumeBilling}'s `already_canceled`). The + * account still restores — the user's data is theirs regardless — but the + * response says billing did not come back, so a caller can say so plainly + * instead of leaving them to discover it on a failed feature. + * + * Sessions are NOT restored (they were deleted, not tombstoned): a restored + * user signs in again. That is the safe direction and it costs one magic link. + */ +export async function handleAccountDeleteUndo( + env: Env, + req: Request, + deps: OAuthDeps, + overrides?: BillingOverrides, +): Promise { + let raw: string | null = null; + if (req.method === "GET") { + raw = new URL(req.url).searchParams.get("token"); + } else { + const body = await readJsonBody(req); + raw = typeof body.token === "string" ? body.token : null; + } + if (!raw) return invalidUndoToken(); + + const hash = await sha256Hex(raw); + const row = await env.DB + .prepare("SELECT id FROM users WHERE delete_undo_hash = ?") + .bind(hash) + .first<{ id: string }>(); + if (!row) return invalidUndoToken(); + const user = await getUserById(env.DB, row.id); + if (!user || user.deletedAt === null) return invalidUndoToken(); + + const now = deps.now?.() ?? new Date(); + const requestedAtMs = Date.parse(user.deletedAt); + // An unparseable timestamp is corruption, not an open window — refuse to + // treat it as restorable rather than guessing which side of the line it's on. + if (!Number.isFinite(requestedAtMs) || now.getTime() - requestedAtMs >= DELETE_UNDO_WINDOW_MS) { + return deleteError( + 410, + "undo_window_expired", + "The 24-hour window to restore this account has passed and its data has been deleted.", + ); + } + + const stripe = stripeFor(env, overrides); + let billingResumed = true; + let billingNote: "already_canceled" | null = null; + if (user.stripeSubscriptionId !== null) { + if (!stripe) { + return deleteError( + 503, + "billing_not_configured", + "Billing is not configured here, so this account's subscription cannot be resumed. The account was not restored; retry later.", + ); + } + try { + const result = await resumeBilling(stripe, user.stripeSubscriptionId); + if (!result.resumed) { + billingResumed = false; + billingNote = result.reason; + } + } catch (err) { + console.error( + `event=account_undo_billing_failed user=${user.id} error=${err instanceof Error ? err.message : String(err)}`, + ); + return deleteError( + 502, + "billing_resume_failed", + "Billing could not be resumed, so the account was not restored. Nothing changed; retry the request.", + ); + } + } + + const res = await env.DB + .prepare( + "UPDATE users SET deleted_at = NULL, delete_undo_hash = NULL, delete_notice_sent_at = NULL WHERE id = ? AND deleted_at IS NOT NULL", + ) + .bind(user.id) + .run(); + // Zero rows means the sweep or a concurrent undo got here first — the token + // is spent either way, and it is now indistinguishable from any other dead + // token, which is exactly the answer it should get. + if ((res.meta.changes ?? 0) === 0) return invalidUndoToken(); + + console.log(`event=account_delete_undone user=${user.id} billing_resumed=${billingResumed}`); + return jsonResponse( + { + restored: true, + billing_resumed: billingResumed, + ...(billingNote ? { billing_note: billingNote } : {}), + }, + 200, + { "cache-control": "no-store" }, + ); +} + +// --- A-4: the convergence sweep ---------------------------------------------- + +/** + * How many failed convergence passes an account may accumulate before the + * sweep stops retrying quietly and pages the operator. At one pass per hour + * this is two days — long enough that a transient DO or Stripe fault drains on + * its own without waking anyone, short enough that the gap between what the + * deletion email promised ("permanently deleted" after 24 hours) and what is + * actually still on disk never quietly becomes a month. + */ +export const ACCOUNT_PURGE_ALERT_AFTER_ATTEMPTS = 48; + +/** + * Record one failed convergence pass and, past the threshold, page the + * operator. Extracted so every `continue` in the sweep goes through the same + * bookkeeping — an unrecorded deferral is exactly the invisible-forever case + * this exists to prevent. + * + * WHY THIS NEEDS A HUMAN AND ISN'T JUST A RETRY. Billing tears down on the + * FIRST pass and its converged marker is durable, so a permanently wedged + * vault leaves an account that pays nothing, appears nowhere in the product, + * and still holds tenant data the user was told in writing was destroyed a day + * ago. Nothing about that state degrades further, and nothing about it + * surfaces — the sweep would go on failing identically, forever, at the same + * log level. The alert is the only thing that converts it into work. + * + * Best-effort throughout: neither the counter write nor the alert may throw + * into the sweep loop, because the account after this one still needs its + * pass. + */ +async function deferAccount( + env: Env, + sender: EmailSender | undefined, + user: User, + now: Date, + reason: string, +): Promise { + const attempts = user.deletePurgeAttempts + 1; + console.error(`event=account_purge_deferred user=${user.id} reason=${reason} attempts=${attempts}`); + try { + await env.DB + .prepare("UPDATE users SET delete_purge_attempts = ? WHERE id = ?") + .bind(attempts, user.id) + .run(); + } catch (err) { + console.error( + `event=account_purge_attempt_write_failed user=${user.id} error=${err instanceof Error ? err.message : String(err)}`, + ); + } + if (attempts < ACCOUNT_PURGE_ALERT_AFTER_ATTEMPTS) return; + + const requestedAt = user.deletedAt ?? "unknown"; + await raiseOpsAlert(env, sender, { + // Keyed per account: one wedged vault must not dedupe away another + // account's alert. The hourly dedupe then applies per account. + key: `account-purge:${user.id}`, + subject: `account deletion stuck after ${attempts} passes`, + text: [ + `An account past its 24-hour delete window has failed to converge for ${attempts} sweep passes.`, + ``, + ` account: ${user.id}`, + ` requested at: ${requestedAt}`, + ` last reason: ${reason}`, + ``, + `This account's billing is already torn down, so it is paying nothing and`, + `is invisible in the product — but its data is STILL PRESENT, and the`, + `deletion email told the user it would be permanently deleted 24 hours`, + `after the request. That promise is currently unmet and will stay unmet`, + `until someone acts.`, + ``, + `Most likely cause: a vault Durable Object that will not answer`, + `POST /api/internal/destroy. Check the event=account_purge_vault_failed`, + `lines for this account to see which vault and why.`, + ``, + `Alert repeats at most hourly per account while the condition persists.`, + ].join("\n"), + now, + }); +} + +export interface AccountDeleteSweepSummary { + /** Accounts past their window that this pass looked at. */ + due: number; + /** Accounts fully converged: billing gone, vaults erased, rows purged. */ + purged: number; + /** Accounts left for the next pass because a step did not converge. */ + deferred: number; + /** Accounts that threw — counted separately from an orderly deferral. */ + failed: number; +} + +/** + * Purge every identity row an account owns, ending with the account itself. + * One D1 batch (D1's atomic unit — there are no interactive transactions), in + * dependency order: the `owners` row resolves through `users.owner_id`, so it + * must go before the row it reads. + * + * Releasing the `owners` row is deliberate, not incidental: a handle is a + * global namespace claim, and leaving it held by a deleted account would + * permanently burn the name. + * + * The vault-scoped tables (`vaults`, `vault_usage`, `vault_snapshots`) are + * absent here on purpose — {@link deleteVaultD1Rows} already removed them + * per-vault, gated on that vault's storage actually being destroyed. Dropping + * them here too would let a vault whose DO destroy FAILED lose its ownership + * row anyway, which is precisely how orphaned, still-billed storage is made. + */ +async function purgeAccountRows(db: D1Database, userId: string): Promise { + await db.batch([ + db.prepare("DELETE FROM sessions WHERE user_id = ?").bind(userId), + db.prepare("DELETE FROM tokens WHERE user_id = ?").bind(userId), + db.prepare("DELETE FROM grants WHERE user_id = ?").bind(userId), + db.prepare("DELETE FROM auth_codes WHERE user_id = ?").bind(userId), + db.prepare("DELETE FROM magic_links WHERE user_id = ?").bind(userId), + db.prepare("DELETE FROM pending_logins WHERE user_id = ?").bind(userId), + db.prepare("DELETE FROM drip_sends WHERE user_id = ?").bind(userId), + db.prepare("DELETE FROM user_checklist WHERE user_id = ?").bind(userId), + db.prepare("DELETE FROM promo_redemptions WHERE user_id = ?").bind(userId), + db + .prepare("DELETE FROM owners WHERE kind = 'user' AND owner_id = (SELECT owner_id FROM users WHERE id = ?)") + .bind(userId), + db.prepare("DELETE FROM users WHERE id = ?").bind(userId), + ]); +} + +/** + * Converge every account whose 24-hour window has closed. Rides the hourly + * tick (ops.ts) rather than claiming a cron pattern of its own — the window is + * a day long, so hourly resolution is ample and a new pattern would have to be + * added to two wrangler.toml `[triggers]` blocks to no benefit. + * + * Per account, in this order, stopping at the first step that does not + * converge: + * + * 1. {@link teardownBilling} — money first. Its NULLed Stripe ids are the + * converged marker, so a retry after a partial pass costs nothing. + * 2. Every owned vault: `callVaultDestroy` (the DO's SQLite and the whole + * `vault-/` R2 prefix), then `deleteVaultD1Rows` — the SAME pair + * the single-vault delete door drives, in the same destroy-then-D1 order, + * so a failed destroy always leaves the ownership row behind to retry + * from. A vault that fails defers the WHOLE account: the user row must + * not be purged while a vault it owns still exists, because the row is + * the only thing that still says whose it was. + * 3. {@link purgeAccountRows} — the account itself, last. + * + * Between passes the account stays tombstoned, which means A-1 keeps refusing + * every read of it. So a deferral is never an exposure: it is a still-closed + * account with residue behind it. + * + * PERMANENT DEFERRAL is the failure mode that actually needs handling, and it + * does not look like a failure from inside the loop. Billing converges on the + * FIRST pass and stays converged, so an account with one permanently wedged + * vault settles into a steady state: paying nothing, absent from every product + * surface, and still holding the data we told the user in writing would be + * permanently deleted 24 hours after they asked. Nothing degrades, nothing + * escalates, and every pass logs the same line. So each deferral increments + * `users.delete_purge_attempts` (migration 0024) and past + * {@link ACCOUNT_PURGE_ALERT_AFTER_ATTEMPTS} passes it pages the operator — + * see {@link deferAccount}. The counter is the only reason this state is ever + * seen by a human. + * + * Failure isolation is per account (the snapshot sweep's posture): one + * account's Stripe outage or wedged DO must not stop the rest of the queue. + */ +export async function runAccountDeleteSweep( + env: Env, + deps: OAuthDeps, + now: Date = new Date(), + overrides?: BillingOverrides & { sender?: EmailSender }, +): Promise { + const cutoff = new Date(now.getTime() - DELETE_UNDO_WINDOW_MS).toISOString(); + const res = await env.DB + .prepare( + `SELECT id FROM users + WHERE deleted_at IS NOT NULL AND deleted_at <= ? + ORDER BY deleted_at ASC LIMIT ?`, + ) + .bind(cutoff, ACCOUNT_DELETE_SWEEP_CAP) + .all<{ id: string }>(); + const due = res.results ?? []; + const summary: AccountDeleteSweepSummary = { due: due.length, purged: 0, deferred: 0, failed: 0 }; + + for (const { id: userId } of due) { + try { + const user = await getUserById(env.DB, userId); + if (!user) continue; // Raced with another pass — already gone. + + // 1. Billing. + if (hasBillingArtifacts(user)) { + const stripe = stripeFor(env, overrides); + if (!stripe) { + await deferAccount(env, overrides?.sender, user, now, "billing_not_configured"); + summary.deferred++; + continue; + } + const billing = await teardownBilling(stripe, env.DB, userId); + if (!billing.converged) { + await deferAccount(env, overrides?.sender, user, now, `billing:${billing.error}`); + summary.deferred++; + continue; + } + } + + // 2. Vault storage — the only step that erases tenant content. + // + // The destroy REPLY is validated, not just its status, through the same + // `readDestroyOutcome` the single-vault door uses (vault-call.ts). This + // is not defensive padding: the very next statement deletes the D1 rows + // that are the only remaining record of whose bytes those were, so a 200 + // that isn't the DO's real answer would orphan the storage permanently — + // still billed, no longer attributable, and beyond the reach of any + // retry, because nothing left in the system would know to try. + let storageConverged = true; + for (const vault of await listVaultsForOwner(env.DB, userId)) { + let outcome: Awaited>; + try { + outcome = await readDestroyOutcome(await callVaultDestroy(env.DB, deps, userId, vault.name)); + } catch (err) { + outcome = { ok: false, reason: "transport", detail: err instanceof Error ? err.message : String(err) }; + } + if (!outcome.ok) { + console.error( + `event=account_purge_vault_failed user=${userId} vault=${vault.name} reason=${outcome.reason} detail=${JSON.stringify(outcome.detail)}`, + ); + storageConverged = false; + continue; + } + await deleteVaultD1Rows(env.DB, userId, vault.name, now); + } + if (!storageConverged) { + await deferAccount(env, overrides?.sender, user, now, "vault_storage"); + summary.deferred++; + continue; + } + + // 3. The account. + await purgeAccountRows(env.DB, userId); + summary.purged++; + console.log(`event=account_purged user=${userId}`); + } catch (err) { + console.error( + `event=account_purge_failed user=${userId} error=${err instanceof Error ? err.message : String(err)}`, + ); + summary.failed++; + } + } + + console.log( + `event=account_delete_sweep due=${summary.due} purged=${summary.purged} deferred=${summary.deferred} failed=${summary.failed}`, + ); + return summary; +} diff --git a/workers/identity/src/account-descriptor.ts b/workers/identity/src/account-descriptor.ts index 9d43c17..7d5d070 100644 --- a/workers/identity/src/account-descriptor.ts +++ b/workers/identity/src/account-descriptor.ts @@ -82,8 +82,20 @@ export function accountDescriptor(deps: OAuthDeps): Response { // APP_CLIENT_ID constant stays exported below — the C5 seeded client and // any cross-origin native flow still use it; only the advertisement goes. // Cloud v1: create yes; rename NO (the vault name is the immutable global - // slug / DO address / URL); delete not yet (handleAccountVaultDelete is 501). - capabilities: { vault_create: true, vault_rename: false, vault_delete: false }, + // slug / DO address / URL); delete YES since cloud#226 — `DELETE + // /account/vaults/` is a real teardown (vault-worker destroy, then the + // identity-side D1 sweep), so the door now advertises the capability it + // actually has. A client that reads this flag is what turns the endpoint + // into a user-facing affordance; leaving it false would ship the route dark. + // NOT advertised here, deliberately: the WHOLE-ACCOUNT door (cloud#226 + // A-3, `/account/delete` + `/account/undo-delete`) has no flag, because + // `AccountCapabilities` is the SHARED door contract — it lives upstream in + // parachute-hub's `packages/door-contract` and is CI-pinned to a reviewed + // commit, so a new key belongs to an upstream PR + a pin bump, not to a + // cloud-side wiring branch. Until that lands, a client learns the door + // exists from the contract's route list, not from a capability bit. The + // route works either way; what it lacks is the advertisement. + capabilities: { vault_create: true, vault_rename: false, vault_delete: true }, // The account-level MCP endpoint (Wave A PR3) — one connection across the // account's vaults (list-vaults / create-vault / query-notes). Advertised at // the FRONT DOOR origin (`my.` in prod, self-referential in staging), where diff --git a/workers/identity/src/index.ts b/workers/identity/src/index.ts index f336cf6..b983bfd 100644 --- a/workers/identity/src/index.ts +++ b/workers/identity/src/index.ts @@ -79,6 +79,7 @@ import { handleAccountVaultTokenMint, handleAccountVaultsList, } from "./account-api.ts"; +import { handleAccountDelete, handleAccountDeleteUndo } from "./account-delete.ts"; import { handleAccountBillingCheckoutPost, handleAccountBillingPortalPost, @@ -254,13 +255,35 @@ app.post("/account/token", (c) => handleAccountToken(c.env.DB, c.req.raw, depsFo // token above (validateAccountToken + hasAccountScope — read for GET, admin for // mutations), account id from the TOKEN not the body. GET list, POST create // (returns a ready vault_token — lands the app IN the vault), per-vault mint, -// and DELETE (501 — no delete door on the hosted side yet). account-api.ts. +// and DELETE (cloud#226 — the real teardown: confirm-retype + ownership gate, +// then the vault worker's destroy, then the identity D1 sweep). account-api.ts. app.get("/account/session", (c) => handleAccountSession(c.env.DB, c.req.raw, depsFor(c.env))); app.get("/account/summary", (c) => handleAccountSummary(c.env.DB, c.req.raw, depsFor(c.env))); app.get("/account/vaults", (c) => handleAccountVaultsList(c.env.DB, c.req.raw, depsFor(c.env))); app.post("/account/vaults", (c) => handleAccountVaultCreate(c.env.DB, c.req.raw, depsFor(c.env))); app.post("/account/vaults/:name/token", (c) => handleAccountVaultTokenMint(c.env.DB, c.req.raw, depsFor(c.env))); app.delete("/account/vaults/:name", (c) => handleAccountVaultDelete(c.env.DB, c.req.raw, depsFor(c.env))); +// ACCOUNT deletion (cloud#226 A-3) — the whole-account sibling of the vault +// delete above, and a different shape on purpose: it severs auth immediately +// (tombstone + session/token wipe) and pauses billing, but erases NOTHING for +// 24 hours. /account/undo-delete is the way back, authenticated by the mailed +// token alone — no bearer, no cookie, both of which the tombstone now refuses +// (migration 0023's chokepoints). Past the window the hourly sweep +// (account-delete.ts, ops.ts) tears down billing, destroys every owned vault, +// and purges the account row. +// +// WHY `/account/delete` AND NOT A BARE `DELETE /account`: the bare `/account` +// path is the Parachute App's own Account-manager SCREEN and is deliberately +// SPA-owned (route-manifest.ts SUBTREE_ONLY_PREFIXES — only `/account/…` is +// worker-first). `run_worker_first` matches on PATH, not method, so registering +// any verb on the bare path would drag the screen into the worker and 404 a +// cold hard-load of it. Both verbs answer here so a client can spell it either +// way; GET and POST both answer undo — GET is the link in the notice email, +// POST is for API clients. +app.delete("/account/delete", (c) => handleAccountDelete(c.env, c.req.raw, depsFor(c.env), senderFor(c.env))); +app.post("/account/delete", (c) => handleAccountDelete(c.env, c.req.raw, depsFor(c.env), senderFor(c.env))); +app.get("/account/undo-delete", (c) => handleAccountDeleteUndo(c.env, c.req.raw, depsFor(c.env))); +app.post("/account/undo-delete", (c) => handleAccountDeleteUndo(c.env, c.req.raw, depsFor(c.env))); // Handles (migration 0022, the GitHub owner-model): claim ONE global handle for // the account. GET reads the claimed handle + an email-derived suggestion // (read); GET /check tests a candidate's availability (read — public info, but diff --git a/workers/identity/src/ops-alerts.ts b/workers/identity/src/ops-alerts.ts new file mode 100644 index 0000000..35331fb --- /dev/null +++ b/workers/identity/src/ops-alerts.ts @@ -0,0 +1,104 @@ +/** + * The operator-alert seam — the "wake someone up" channel, extracted from + * ops.ts so surfaces OTHER than the health check can use it without importing + * ops.ts (which imports them back; the cron router and the jobs it routes to + * would form a cycle). + * + * The mechanism is unchanged from the health check that has always owned it: + * a `ops_alerts` row per alert KEY holding the last time an email actually + * went out, read-before-send and upserted-after-send, so a persistent fault + * pages once an hour rather than every tick. + * + * WHAT BELONGS HERE, and it is a narrow set: a condition that (a) no user + * action will resolve, (b) leaves the system in a state its own retries do not + * drain, and (c) is invisible in the product. A structured `console.error` is + * the always-on trail for everything else — Workers Logs is queryable, and + * most failures are either self-healing or surfaced to the caller that caused + * them. Emailing on those would train the operator to ignore the channel, + * which costs more than the alerts are worth. + */ +import type { EmailSender } from "./email.ts"; +import type { Env } from "./env.ts"; + +/** Re-alert at most once per hour per key. */ +export const ALERT_DEDUPE_MS = 60 * 60 * 1000; + +/** + * Whether `key` is outside its dedupe window. FAILS OPEN on a D1 error — if D1 + * is down we can't read the dedupe row, and alerting every tick for the + * duration of an outage beats suppressing the one email that matters. + */ +export async function shouldAlert(db: D1Database, key: string, now: Date): Promise { + try { + const row = await db + .prepare("SELECT last_alert_at FROM ops_alerts WHERE key = ?") + .bind(key) + .first<{ last_alert_at: string }>(); + if (row && now.getTime() - Date.parse(row.last_alert_at) < ALERT_DEDUPE_MS) return false; + return true; + } catch { + return true; + } +} + +export async function markAlerted(db: D1Database, key: string, now: Date): Promise { + try { + await db + .prepare( + "INSERT INTO ops_alerts (key, last_alert_at) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET last_alert_at = excluded.last_alert_at", + ) + .bind(key, now.toISOString()) + .run(); + } catch (err) { + // Best-effort: a failed mark means at worst an extra alert next run. + console.error(`event=ops_alert_mark_failed key=${key} error=${JSON.stringify(String(err))}`); + } +} + +/** + * Raise one deduped operator alert. Returns whether an email actually went + * out, so a caller can log the difference between "condition seen" and + * "operator told". + * + * NEVER THROWS. Every call site is already handling a failure, and an alert + * that escalates into a second exception would replace a recoverable problem + * with an unhandled one — in a cron tick, that means the jobs after it never + * run. A send failure is logged and swallowed. + * + * The structured log is emitted regardless of dedupe or delivery, so the trail + * is complete even when the email is suppressed. + */ +export async function raiseOpsAlert( + env: Env, + sender: EmailSender | undefined, + opts: { key: string; subject: string; text: string; now: Date }, +): Promise { + console.error(`event=ops_alert key=${opts.key} detail=${JSON.stringify(opts.subject)}`); + try { + if (!sender) return false; + const to = env.OPERATOR_ALERT_EMAIL; + if (!to) { + console.error("event=ops_alert_skipped reason=no_operator_alert_email"); + return false; + } + if (!(await shouldAlert(env.DB, opts.key, opts.now))) return false; + + const envName = env.ENVIRONMENT ?? "unknown"; + const result = await sender.sendOps({ + to, + subject: `[parachute-cloud ${envName}] ${opts.subject}`, + text: opts.text, + }); + if (!result.ok) { + console.error(`event=ops_alert_send_failed key=${opts.key} error=${JSON.stringify(result.error)}`); + return false; + } + await markAlerted(env.DB, opts.key, opts.now); + return true; + } catch (err) { + console.error( + `event=ops_alert_raise_failed key=${opts.key} error=${err instanceof Error ? err.message : String(err)}`, + ); + return false; + } +} diff --git a/workers/identity/src/ops.ts b/workers/identity/src/ops.ts index b3d8311..e66e363 100644 --- a/workers/identity/src/ops.ts +++ b/workers/identity/src/ops.ts @@ -17,7 +17,11 @@ * - hourly at :15 (DRIP_CRON): the onboarding email drip — routed here, * implemented in drip.ts (eligibility windows, idempotence ledger, * per-run cap, unsubscribe) — plus the billing sweep (Wave 4d, - * billing-lifecycle.ts): apply due pending plan downgrades. + * billing-lifecycle.ts): apply due pending plan downgrades — plus the + * account-delete convergence sweep (cloud#226 A-4, account-delete.ts): + * for accounts past their 24-hour undo window, tear down billing, destroy + * every owned vault, and purge the account rows. Three jobs, one tick, + * three independent try blocks — they must not share a failure. * - daily 03:30 UTC (USAGE_CRON): the per-vault storage-usage rollup plus * plan-entitlement reconciler — routed here, implemented in usage.ts * (one internal-config GET and one D1 `vault_usage` row per vault per UTC @@ -32,6 +36,8 @@ */ import type { Env } from "./env.ts"; import type { EmailSender } from "./email.ts"; +import { runAccountDeleteSweep } from "./account-delete.ts"; +import { markAlerted, shouldAlert } from "./ops-alerts.ts"; import { runBillingSweep } from "./billing-lifecycle.ts"; import { runDrip } from "./drip.ts"; import { depsForEnv } from "./oauth-shared.ts"; @@ -48,8 +54,9 @@ export const USAGE_CRON = "30 3 * * *"; /** Nightly GFS snapshot sweep (snapshots.ts) — 04:00 UTC, after the usage rollup. */ export const SNAPSHOT_CRON = "0 4 * * *"; -/** Re-alert at most once per hour per failing check. */ -export const ALERT_DEDUPE_MS = 60 * 60 * 1000; +/** Re-alert at most once per hour per failing check. Re-exported from the + * shared alert seam (ops-alerts.ts) — existing importers keep working. */ +export { ALERT_DEDUPE_MS } from "./ops-alerts.ts"; /** Budget for the vault /health round-trip before we call it down. */ const HEALTH_FETCH_TIMEOUT_MS = 10_000; @@ -80,7 +87,19 @@ export async function handleScheduled(cron: string, env: Env, sender: EmailSende if (job === "digest") { await sendWeeklyDigest(env, sender, deps); } else if (job === "drip") { - await runDrip(env, sender, deps); + // GUARDED like its two tick-mates, and for a reason the comment below only + // half-stated until cloud#226: an unguarded `await` here does not merely + // lose the drip, it starves BOTH sweeps forever. A permanently-throwing + // drip (a template bug, a sender outage that escapes runDrip's own + // handling) would mean no plan downgrade is ever applied and no deleted + // account is ever purged — the second of which silently breaks a promise + // made in writing to a user ("permanently deleted after 24 hours"). + // Nothing downstream depends on the drip having run. + try { + await runDrip(env, sender, deps); + } catch (err) { + console.error(`event=drip_failed error=${err instanceof Error ? err.message : String(err)}`); + } // The billing sweep rides the same hourly tick (no new cron pattern): // apply due pending downgrades (billing-lifecycle.ts — pending_plan + // plan_downgrade_at, stamped by customer.subscription.deleted). Guarded @@ -92,6 +111,22 @@ export async function handleScheduled(cron: string, env: Env, sender: EmailSende } catch (err) { console.error(`event=billing_sweep_failed error=${err instanceof Error ? err.message : String(err)}`); } + // The account-delete convergence sweep (cloud#226 A-4, account-delete.ts) + // rides the same hourly tick — the undo window is 24 hours, so hourly + // resolution is ample and a dedicated cron pattern would have to be added + // to two wrangler.toml [triggers] blocks for nothing. Independently + // guarded, in its own try, for the same reason the billing sweep is: these + // three jobs share a tick but must not share a failure. It goes LAST + // because it is the only one that destroys anything. + try { + const purgeDeps = depsForEnv(env); + if (deps.now) purgeDeps.now = deps.now; + // The sender goes in so a permanently-stuck purge can page the operator + // (ops-alerts.ts) — see runAccountDeleteSweep's PERMANENT DEFERRAL note. + await runAccountDeleteSweep(env, purgeDeps, deps.now?.() ?? new Date(), { sender }); + } catch (err) { + console.error(`event=account_delete_sweep_failed error=${err instanceof Error ? err.message : String(err)}`); + } } else if (job === "usage") { // The rollup's vault reads go through the mint seam, so it takes the full // OAuthDeps (issuer/signing + per-environment transport), not OpsDeps. @@ -147,27 +182,7 @@ export async function checkVaultHealth(vaultOrigin: string, fetchFn: typeof fetc * is down we can't read the dedupe row — better to alert every 10 minutes for * the duration of a D1 outage than to suppress the one email that matters. */ -async function shouldAlert(db: D1Database, key: string, now: Date): Promise { - try { - const row = await db.prepare("SELECT last_alert_at FROM ops_alerts WHERE key = ?").bind(key).first<{ last_alert_at: string }>(); - if (row && now.getTime() - Date.parse(row.last_alert_at) < ALERT_DEDUPE_MS) return false; - return true; - } catch { - return true; - } -} -async function markAlerted(db: D1Database, key: string, now: Date): Promise { - try { - await db - .prepare("INSERT INTO ops_alerts (key, last_alert_at) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET last_alert_at = excluded.last_alert_at") - .bind(key, now.toISOString()) - .run(); - } catch (err) { - // Best-effort: a failed mark means at worst an extra alert next run. - console.error(`event=ops_alert_mark_failed key=${key} error=${JSON.stringify(String(err))}`); - } -} /** * Run both checks; email the operator about any failure that hasn't alerted in diff --git a/workers/identity/src/users.ts b/workers/identity/src/users.ts index 5f39cc3..5247f0b 100644 --- a/workers/identity/src/users.ts +++ b/workers/identity/src/users.ts @@ -82,8 +82,16 @@ export interface User { * delete route (A-3) starts writing it. */ deleteUndoHash: string | null; /** ISO-8601 timestamp the deletion-notice email was sent (migration 0023). - * Unused until the delete route (A-3) starts writing it. */ + * Set by the delete route (A-3) on a real send only. */ deleteNoticeSentAt: string | null; + /** + * How many convergence-sweep passes have tried and failed to finish this + * account's teardown (migration 0024). NULL in storage means "never + * deferred" and surfaces here as 0, so callers never branch on the + * null-vs-zero distinction — there isn't one. Reset is unnecessary: an + * account that converges has its row deleted. + */ + deletePurgeAttempts: number; /** * Stripe linkage (migration 0012) — set by the checkout.session.completed * webhook; null for free users and comped accounts. The lifecycle handlers @@ -125,6 +133,7 @@ interface Row { deleted_at: string | null; delete_undo_hash: string | null; delete_notice_sent_at: string | null; + delete_purge_attempts: number | null; stripe_customer_id: string | null; stripe_subscription_id: string | null; pending_plan: string | null; @@ -150,6 +159,7 @@ function rowToUser(r: Row): User { deletedAt: r.deleted_at, deleteUndoHash: r.delete_undo_hash, deleteNoticeSentAt: r.delete_notice_sent_at, + deletePurgeAttempts: r.delete_purge_attempts ?? 0, stripeCustomerId: r.stripe_customer_id, stripeSubscriptionId: r.stripe_subscription_id, // Same defensive coercion as `plan`; null stays null (no pending change). @@ -282,6 +292,7 @@ export async function createUser( deletedAt: null, deleteUndoHash: null, deleteNoticeSentAt: null, + deletePurgeAttempts: 0, stripeCustomerId: null, stripeSubscriptionId: null, pendingPlan: "expired", diff --git a/workers/identity/src/vault-call.ts b/workers/identity/src/vault-call.ts index 195174e..9676784 100644 --- a/workers/identity/src/vault-call.ts +++ b/workers/identity/src/vault-call.ts @@ -182,6 +182,92 @@ export async function callVaultImport( }); } +/** + * Ask the target vault's DO to perform its irreversible teardown + * (`POST /api/internal/destroy`). This uses the same first-party admin mint as + * the config, snapshot, restore, and import seams; the identity-side ownership + * check happens before this helper is called, so this function only owns the + * issuer→vault hop. + * + * NOT best-effort: the account delete route must inspect the returned status and + * body before it touches D1. A transport error propagates, and a non-2xx response + * is returned to the caller rather than being swallowed like `pushVaultCap`'s + * plan-reconciliation failures. That ordering is the safe direction: a failed + * storage teardown must leave the ownership and accounting rows available for a + * retry. + */ +export async function callVaultDestroy( + db: D1Database, + deps: OAuthDeps, + userId: string, + vaultName: string, +): Promise { + return callVaultApi(db, deps, { + userId, + vaultName, + method: "POST", + apiPath: "/api/internal/destroy", + verb: "admin", + jsonBody: { confirm: vaultName }, + }); +} + +/** + * Why a destroy could not be believed, or the count of R2 objects it removed. + * `reason` is a log/message token, never shown raw to a tenant. + */ +export type DestroyOutcome = + | { ok: true; r2ObjectsDeleted: number } + | { ok: false; reason: "transport" | "status" | "unreadable" | "malformed"; detail: string }; + +/** + * THE ONLY PLACE A DESTROY RESPONSE IS BELIEVED. Both callers of + * {@link callVaultDestroy} — the single-vault door (account-api.ts) and the + * account sweep (account-delete.ts) — go through here, because the thing they + * do next is delete the D1 rows that are the ONLY remaining record of which + * tenant those bytes belonged to. If that record goes while the bytes stay, + * the storage is orphaned: still billed, no longer attributable, and not + * reachable by any retry, because nothing left in the system knows to try. + * + * So a 200 is NOT sufficient. The body must be the DO's real reply — a + * literal `destroyed: true` plus an `r2_objects_deleted` that is a safe, + * non-negative integer. A 200 carrying anything else (a renamed route + * answering generically, a service-binding or proxy quirk, a future handler + * that returns `{ok:true}`) is treated exactly like a failed destroy: the + * caller must leave every row in place. + * + * The asymmetry is deliberate and is the whole point. A false negative costs + * one retry of an idempotent call. A false positive is unrecoverable. + * + * Takes the already-resolved `Response` (or the thrown error) rather than + * making the call itself, so each caller keeps its own control flow — the + * door answers a tenant with an HTTP status, the sweep defers an account — + * while sharing the one judgement that must not diverge between them. + */ +export async function readDestroyOutcome(res: Response): Promise { + if (!res.ok) return { ok: false, reason: "status", detail: `HTTP ${res.status}` }; + let body: { destroyed?: unknown; r2_objects_deleted?: unknown }; + try { + body = (await res.json()) as { destroyed?: unknown; r2_objects_deleted?: unknown }; + } catch { + return { ok: false, reason: "unreadable", detail: "response body was not JSON" }; + } + const count = body.r2_objects_deleted; + if ( + body.destroyed !== true || + typeof count !== "number" || + !Number.isSafeInteger(count) || + count < 0 + ) { + return { + ok: false, + reason: "malformed", + detail: `destroyed=${JSON.stringify(body.destroyed)} r2_objects_deleted=${JSON.stringify(count)}`, + }; + } + return { ok: true, r2ObjectsDeleted: count }; +} + /** The DO's resolved plan entitlement, as reported by GET /api/internal/config. */ export interface ResolvedVaultEntitlement { /** The currently resolved two-meter entitlement; null means unpushed (a diff --git a/workers/identity/src/vaults.ts b/workers/identity/src/vaults.ts index 342bb92..3468c96 100644 --- a/workers/identity/src/vaults.ts +++ b/workers/identity/src/vaults.ts @@ -133,6 +133,161 @@ export async function userOwnsVault(db: D1Database, userId: string, name: string return v !== null && v.ownerUserId === userId; } +export interface VaultDeleteD1Summary { + vaultRowsDeleted: number; + usageRowsDeleted: number; + snapshotRowsDeleted: number; + tokensRevoked: number; + grantsRewritten: number; + grantsDropped: number; +} + +/** + * A LIKE pattern matching any scope string that CONTAINS the `vault::` + * segment prefix — the cheap D1-side CANDIDATE filter for the delete cascade, + * never the authority (see {@link deleteVaultD1Rows}). Wildcards in the name are + * escaped (`ESCAPE '\'`) because `_` is a LIKE single-char wildcard and legacy + * rows can predate today's stricter slug gate. + */ +function vaultScopeLikePattern(vaultName: string): string { + return `%vault:${vaultName.replace(/[\\%_]/g, "\\$&")}:%`; +} + +/** The scope strings in a space-delimited scope column, as an array. */ +function splitScopes(scopes: string): string[] { + return scopes.split(" ").filter((s) => s.length > 0); +} + +/** + * Remove the identity-side rows for one owned vault after the vault worker has + * erased the DO and its R2 prefix. Ports the self-hosted hub twin's cascade + * (`handleDeleteVault` → `revokeTokensNamingVault` + `rewriteGrantsRemovingVault`) + * onto D1, in the same order — identity artifacts first, ownership claim last, + * because revocation is the safe direction if a later step fails. + * + * MATCHING IS EXACT SCOPE-SEGMENT COMPARISON, NEVER `LIKE` — the twin's rule, + * kept here. A scope names this vault iff it parses as the three-part + * `vault::` grammar ({@link vaultScopeName}) with `` equal; + * a substring hit is not enough. The one adaptation for D1 is that a `LIKE` + * pattern PRE-FILTERS which rows are read (the hub reads the whole unrevoked + * registry into memory; cloud's is multi-tenant and must not). That is sound in + * exactly one direction: every true match contains the `vault::` substring, + * so the pattern is a superset — it can over-fetch candidates, never miss one, + * and the JS check discards the extras. Do not promote the pattern to the + * decision; `xvault:foo:read` and `account::vaults:foo:read` are precisely + * the strings that must survive it. + * + * Grants are REWRITTEN, not dropped, when they name the vault — a `grants` row + * is keyed (user, client) and its scope set spans every vault that user ever + * approved for that client, so dropping the row over one vault would silently + * revoke the client's consent on the user's OTHER vaults. The row is deleted + * only when the rewrite empties it. This also closes the re-create hole: a + * client that held `vault::read` must face the consent screen again if a + * vault of that name is ever created anew. + * + * Token registry rows are retained and marked `revoked_at`, matching the + * identity worker's existing revocation-list convention. Only rows that were + * still live contribute to `tokensRevoked`, so a retry after a D1 failure is + * idempotent — as are the row deletes. + * + * Every write runs in ONE D1 batch (D1 has no interactive transaction; a batch + * is the atomic unit). A batch failure therefore leaves the whole cascade + * available for a retry, which matters because the preceding vault destroy is + * intentionally irreversible but itself idempotent. The two candidate SELECTs + * run before it and write nothing. + * + * REMAINING RESIDUE, stated rather than implied away: the cascade sweeps the + * three-part `vault::` grammar the twin defines, and NOT cloud's + * own composed ACCOUNT scopes (`account::vaults::`), which have + * no hub counterpart. Those are inert against a deleted vault — the account-MCP + * fan-out resolves vaults through `listVaultsForOwner` at call time, so a dead + * name reaches nothing — but they would still cover a same-name vault created + * later without a fresh consent. Sweeping them safely means reasoning about + * `recordGrant`'s family-replace narrowing semantics (grants.ts), which is a + * consent-model change and wants its own review, not a wiring slice (cloud#226). + */ +export async function deleteVaultD1Rows( + db: D1Database, + ownerUserId: string, + name: string, + now: Date = new Date(), +): Promise { + const vaultName = name.toLowerCase(); + const pattern = vaultScopeLikePattern(vaultName); + + const [tokenCandidates, grantCandidates] = await Promise.all([ + db + .prepare("SELECT jti, scopes FROM tokens WHERE revoked_at IS NULL AND scopes LIKE ? ESCAPE '\\'") + .bind(pattern) + .all<{ jti: string; scopes: string }>(), + db + .prepare("SELECT user_id, client_id, scopes FROM grants WHERE scopes LIKE ? ESCAPE '\\'") + .bind(pattern) + .all<{ user_id: string; client_id: string; scopes: string }>(), + ]); + + const jtis = (tokenCandidates.results ?? []) + .filter((r) => splitScopes(r.scopes).some((s) => vaultScopeName(s) === vaultName)) + .map((r) => r.jti); + + const statements: D1PreparedStatement[] = []; + + // Revoke in chunks: SQLite caps bound parameters (~999), and a busy account + // can hold more live vault tokens than one IN-list should carry. + const REVOKE_CHUNK = 100; + const revokeStatementCount = Math.ceil(jtis.length / REVOKE_CHUNK); + for (let i = 0; i < jtis.length; i += REVOKE_CHUNK) { + const chunk = jtis.slice(i, i + REVOKE_CHUNK); + statements.push( + db + .prepare( + `UPDATE tokens SET revoked_at = ? WHERE revoked_at IS NULL AND jti IN (${chunk.map(() => "?").join(", ")})`, + ) + .bind(now.toISOString(), ...chunk), + ); + } + + let grantsRewritten = 0; + let grantsDropped = 0; + for (const row of grantCandidates.results ?? []) { + const scopes = splitScopes(row.scopes); + const kept = scopes.filter((s) => vaultScopeName(s) !== vaultName); + if (kept.length === scopes.length) continue; // LIKE over-fetch — not a real match. + if (kept.length === 0) { + statements.push( + db.prepare("DELETE FROM grants WHERE user_id = ? AND client_id = ?").bind(row.user_id, row.client_id), + ); + grantsDropped++; + } else { + statements.push( + db + .prepare("UPDATE grants SET scopes = ? WHERE user_id = ? AND client_id = ?") + .bind(kept.join(" "), row.user_id, row.client_id), + ); + grantsRewritten++; + } + } + + const mirrorsAt = statements.length; + statements.push( + db.prepare("DELETE FROM vault_usage WHERE vault_name = ?").bind(vaultName), + db.prepare("DELETE FROM vault_snapshots WHERE vault_name = ?").bind(vaultName), + db.prepare("DELETE FROM vaults WHERE name = ? AND owner_user_id = ?").bind(vaultName, ownerUserId), + ); + + const results = await db.batch(statements); + let tokensRevoked = 0; + for (let i = 0; i < revokeStatementCount; i++) tokensRevoked += results[i]?.meta.changes ?? 0; + return { + tokensRevoked, + grantsRewritten, + grantsDropped, + usageRowsDeleted: results[mirrorsAt]?.meta.changes ?? 0, + snapshotRowsDeleted: results[mirrorsAt + 1]?.meta.changes ?? 0, + vaultRowsDeleted: results[mirrorsAt + 2]?.meta.changes ?? 0, + }; +} + /** * Claim a vault name for a user. Validates slug + reserved, then inserts. Throws * VaultNameInvalidError (bad name) or VaultNameTakenError (name already owned by @@ -193,14 +348,27 @@ export async function clearImportPending(db: D1Database, name: string, ownerUser .run(); } +/** + * The vault a single scope NAMES, or null when it names none. The one place the + * `vault::` grammar is decided — twin of the hub's `vaultScopeName`. + * Returns null for an unnamed vault scope (`vault:read`), for a non-vault scope, + * and for every ACCOUNT scope family (`account::vaults::` has + * five parts and a `vaults` head, so it can never be mistaken for this one). + */ +export function vaultScopeName(scope: string): string | null { + const parts = scope.split(":"); + if (parts.length === 3 && parts[0] === "vault" && parts[1] && parts[2] && VAULT_VERBS.has(parts[2])) { + return parts[1]; + } + return null; +} + /** The distinct named vaults referenced by a scope set (`vault::`). */ export function namedVaultsInScopes(scopes: readonly string[]): string[] { const names = new Set(); for (const s of scopes) { - const parts = s.split(":"); - if (parts.length === 3 && parts[0] === "vault" && parts[1] && parts[2] && VAULT_VERBS.has(parts[2])) { - names.add(parts[1]); - } + const named = vaultScopeName(s); + if (named !== null) names.add(named); } return Array.from(names); } diff --git a/workers/identity/test/account-api.test.ts b/workers/identity/test/account-api.test.ts index 2451176..30eb0bc 100644 --- a/workers/identity/test/account-api.test.ts +++ b/workers/identity/test/account-api.test.ts @@ -9,7 +9,8 @@ * - POST /account/vaults: the hinge — returns a usable vault_token * (aud=vault., read+write, client_id=parachute-account), records * ownership under the TOKEN's account, and refuses at-cap / bad-name / taken; - * - DELETE: 501 (no hosted delete door yet), still admin-gated; + * - DELETE: confirm + ownership-gated destroy-first teardown, D1 cascade, and + * honest failure/idempotency behavior; * - POST /account/vaults//token: owned-only mint (unowned/unknown → one * 403, no oracle), scope-validated (injections 400), default read+write; * - the read-time suspend chokepoint: a suspended owner's token → 401 on the @@ -36,14 +37,19 @@ import type { OAuthDeps } from "../src/oauth-shared.ts"; import { signAccessToken } from "../src/tokens.ts"; import { ISSUER, db, decodeJwtPayload, deps, seedSession, seedUser, seedVault } from "./helpers.ts"; +type VaultFetch = NonNullable; + /** * Deps for the account surface. The create path's `pushVaultCap` PUTs the plan - * cap to the (nonexistent, in-test) vault DO — stub `vaultFetch` 200 so create - * is deterministic and silent. Every other route (list, mint, delete) makes no - * outbound call. + * cap to the (nonexistent, in-test) vault DO — stub `vaultFetch` with a successful + * destroy-shaped response so create is deterministic and silent. Delete tests + * pass their own fetch to inspect or fail that one storage call. */ -function accountDeps(now?: () => Date): OAuthDeps { - return { ...deps(now), vaultFetch: async () => Response.json({ ok: true }, { status: 200 }) }; +function accountDeps( + now?: () => Date, + vaultFetch: VaultFetch = async () => Response.json({ destroyed: true, r2_objects_deleted: 0 }, { status: 200 }), +): OAuthDeps { + return { ...deps(now), vaultFetch }; } /** Mint an account bearer for `userId` with `verb` authority, aud="account". The @@ -90,6 +96,132 @@ async function seedOwnerWithPlan(email: string, plan?: string): Promise<{ userId return { userId: id, token }; } +/** Seed every identity-side artifact the delete cascade owns, plus two token + * rows that pin the registry convention: one live row to revoke and one already + * revoked row that must remain a no-op on the sweep. */ +async function seedDeleteArtifacts(vaultName: string, userId: string): Promise { + const now = new Date("2026-08-10T00:00:00.000Z"); + const expiresAt = new Date(now.getTime() + 60 * 60 * 1000).toISOString(); + await env.DB.batch([ + env.DB + .prepare("INSERT INTO vault_usage (vault_name, day, db_bytes, r2_bytes) VALUES (?, ?, ?, ?)") + .bind(vaultName, "2026-08-08", 10, 20), + env.DB + .prepare("INSERT INTO vault_usage (vault_name, day, db_bytes, r2_bytes) VALUES (?, ?, ?, ?)") + .bind(vaultName, "2026-08-09", 30, 40), + env.DB + .prepare( + "INSERT INTO vault_snapshots (vault_name, key, taken_at, bytes, ranks) VALUES (?, ?, ?, ?, ?)", + ) + .bind(vaultName, `vault-${vaultName}/snapshots/one.tar`, now.toISOString(), 100, "[]"), + env.DB + .prepare( + "INSERT INTO vault_snapshots (vault_name, key, taken_at, bytes, ranks) VALUES (?, ?, ?, ?, ?)", + ) + .bind(vaultName, `vault-${vaultName}/snapshots/two.tar`, now.toISOString(), 200, "[]"), + env.DB + .prepare( + "INSERT INTO tokens (jti, user_id, client_id, scopes, expires_at, revoked_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .bind("delete-live-token", userId, "delete-test", `vault:${vaultName}:read vault:${vaultName}:write`, expiresAt, null, now.toISOString()), + env.DB + .prepare( + "INSERT INTO tokens (jti, user_id, client_id, scopes, expires_at, revoked_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .bind( + "delete-already-revoked-token", + userId, + "delete-test", + `vault:${vaultName}:admin`, + expiresAt, + now.toISOString(), + now.toISOString(), + ), + env.DB + .prepare( + "INSERT INTO tokens (jti, user_id, client_id, scopes, expires_at, revoked_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .bind("delete-other-vault-token", userId, "delete-test", "vault:keep:read", expiresAt, null, now.toISOString()), + // The near-miss the LIKE pattern must NOT catch: a COMPOSED account scope + // naming this same vault. It reads `vaults::` (with the s), so + // `%vault::%` cannot match it — and it must not, because revoking it + // would kill the holder's access to every OTHER vault on the account too. + env.DB + .prepare( + "INSERT INTO tokens (jti, user_id, client_id, scopes, expires_at, revoked_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .bind( + "delete-composed-account-token", + userId, + "delete-test", + `account:${userId}:vaults:${vaultName}:read`, + expiresAt, + null, + now.toISOString(), + ), + // The two near-misses that DO survive the LIKE prefilter (both contain the + // literal `vault::`) and must be thrown out by the exact-segment + // check in JS. These are what go red if `LIKE` is ever promoted from + // candidate filter to decision. + env.DB + .prepare( + "INSERT INTO tokens (jti, user_id, client_id, scopes, expires_at, revoked_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .bind("delete-prefix-suffixed", userId, "delete-test", `xvault:${vaultName}:read`, expiresAt, null, now.toISOString()), + env.DB + .prepare( + "INSERT INTO tokens (jti, user_id, client_id, scopes, expires_at, revoked_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .bind("delete-bogus-verb", userId, "delete-test", `vault:${vaultName}:superuser`, expiresAt, null, now.toISOString()), + ]); + // Standing consent rows, the hub twin's rewrite-don't-drop case: + // - `spanning` names this vault AND another → REWRITTEN, keeping the other; + // - `only-this` names this vault alone → DROPPED; + // - `untouched` names another vault only → left completely alone. + await env.DB.batch([ + env.DB + .prepare("INSERT INTO grants (user_id, client_id, scopes, granted_at) VALUES (?, ?, ?, ?)") + .bind(userId, "grant-spanning", `vault:${vaultName}:read vault:keep:read vault:${vaultName}:write`, now.toISOString()), + env.DB + .prepare("INSERT INTO grants (user_id, client_id, scopes, granted_at) VALUES (?, ?, ?, ?)") + .bind(userId, "grant-only-this", `vault:${vaultName}:read`, now.toISOString()), + env.DB + .prepare("INSERT INTO grants (user_id, client_id, scopes, granted_at) VALUES (?, ?, ?, ?)") + .bind(userId, "grant-untouched", "vault:keep:read", now.toISOString()), + ]); + return now; +} + +async function grantScopes(userId: string, clientId: string): Promise { + const row = await env.DB.prepare("SELECT scopes FROM grants WHERE user_id = ? AND client_id = ?") + .bind(userId, clientId) + .first<{ scopes: string }>(); + return row?.scopes ?? null; +} + +async function deleteArtifactState(vaultName: string): Promise<{ + vaultExists: boolean; + usageRows: number; + snapshotRows: number; + liveTokenRevokedAt: string | null; +}> { + const [vault, usage, snapshots, token] = await Promise.all([ + env.DB.prepare("SELECT 1 AS one FROM vaults WHERE name = ?").bind(vaultName).first<{ one: number }>(), + env.DB.prepare("SELECT COUNT(*) AS n FROM vault_usage WHERE vault_name = ?").bind(vaultName).first<{ n: number }>(), + env.DB + .prepare("SELECT COUNT(*) AS n FROM vault_snapshots WHERE vault_name = ?") + .bind(vaultName) + .first<{ n: number }>(), + env.DB.prepare("SELECT revoked_at FROM tokens WHERE jti = ?").bind("delete-live-token").first<{ revoked_at: string | null }>(), + ]); + return { + vaultExists: vault !== null, + usageRows: usage?.n ?? 0, + snapshotRows: snapshots?.n ?? 0, + liveTokenRevokedAt: token?.revoked_at ?? null, + }; +} + // --- the Bearer gate (every route) ------------------------------------------- describe("C3 — the Bearer gate", () => { @@ -430,36 +562,349 @@ describe("C3 — POST /account/vaults (create lands you IN the vault)", () => { }); }); -// --- DELETE /account/vaults/ — 501 ------------------------------------- +// --- DELETE /account/vaults/ — destroy + identity cascade ---------------- describe("C3 — DELETE /account/vaults/", () => { - test("501 not_implemented (no hosted delete door yet), for an admin token", async () => { - const { userId, token } = await seedOwnerWithPlan("del@example.com"); + test("200 destroys first, then removes identity rows and revokes matching tokens", async () => { + const { userId, token } = await seedOwnerWithPlan("del-success@example.com"); await seedVault("doomed", userId); + const now = await seedDeleteArtifacts("doomed", userId); + const calls: { input: RequestInfo | URL; init?: RequestInit }[] = []; + const vaultFetch: VaultFetch = async (input, init) => { + calls.push({ input, init }); + return Response.json({ destroyed: true, r2_objects_deleted: 7 }, { status: 200 }); + }; + + const res = await handleAccountVaultDelete( + db(), + accountReq("DELETE", "/account/vaults/doomed", { token, body: { confirm: "doomed" } }), + accountDeps(() => now, vaultFetch), + ); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ + destroyed: true, + r2_objects_deleted: 7, + d1: { + vault_rows_deleted: 1, + vault_usage_rows_deleted: 2, + vault_snapshot_rows_deleted: 2, + tokens_revoked: 1, + grants_rewritten: 1, + grants_dropped: 1, + }, + }); + + expect(calls).toHaveLength(1); + const outbound = new Request(calls[0]!.input, calls[0]!.init); + expect(outbound.method).toBe("POST"); + expect(new URL(outbound.url).pathname).toContain("/api/internal/destroy"); + expect(await outbound.json()).toEqual({ confirm: "doomed" }); + const destroyToken = outbound.headers.get("authorization")?.replace(/^Bearer\s+/, ""); + expect(destroyToken).toBeTruthy(); + const destroyClaims = decodeJwtPayload(destroyToken!); + expect(destroyClaims.client_id).toBe("parachute-console"); + expect(destroyClaims.scope).toBe("vault:doomed:admin"); + expect(destroyClaims.aud).toBe("vault.doomed"); + + expect(await deleteArtifactState("doomed")).toEqual({ + vaultExists: false, + usageRows: 0, + snapshotRows: 0, + liveTokenRevokedAt: now.toISOString(), + }); + const unrelated = await env.DB.prepare("SELECT revoked_at FROM tokens WHERE jti = ?") + .bind("delete-other-vault-token") + .first<{ revoked_at: string | null }>(); + expect(unrelated?.revoked_at).toBeNull(); + // The sweep's blast radius stops at the 3-part vault grammar: a composed + // ACCOUNT scope naming the same vault survives. Goes red if the LIKE + // pattern is ever loosened (e.g. to `%%` or `%vault%%`). + // The exact-segment check is the authority, not the LIKE prefilter: the + // composed account scope never reaches it, and the two rows that DO reach + // it (`xvault::read`, `vault::superuser` — both carry the + // literal `vault::` substring) must be discarded by the JS match. + const survivors = await env.DB.prepare( + "SELECT jti FROM tokens WHERE revoked_at IS NULL AND jti IN (?, ?, ?) ORDER BY jti", + ) + .bind("delete-composed-account-token", "delete-prefix-suffixed", "delete-bogus-verb") + .all<{ jti: string }>(); + expect((survivors.results ?? []).map((r) => r.jti)).toEqual([ + "delete-bogus-verb", + "delete-composed-account-token", + "delete-prefix-suffixed", + ]); + + // Grants: REWRITTEN, not dropped, when the row spans other vaults — the hub + // twin's over-revocation guard. Dropping `grant-spanning` would silently + // revoke that client's consent on `keep`, a vault this delete never touched. + expect(await grantScopes(userId, "grant-spanning")).toBe("vault:keep:read"); + expect(await grantScopes(userId, "grant-only-this")).toBeNull(); + expect(await grantScopes(userId, "grant-untouched")).toBe("vault:keep:read"); + }); + + test("the sweep matches scopes by exact segment, not substring — a `_` name cannot catch a neighbor", async () => { + // `_` is a LIKE single-char wildcard, so a naive `LIKE '%vault:my_vault:%'` + // sweep would also revoke `myxvault`-scoped rows. Legacy names can contain + // `_` (they predate today's stricter slug gate), which is exactly why the + // hub twin refuses to let LIKE be the decision. Both rows here would be + // caught by an unescaped pattern; only the real one may be revoked. + const { userId, token } = await seedOwnerWithPlan("del-underscore@example.com"); + await env.DB.prepare("INSERT INTO vaults (name, owner_user_id, created_at) VALUES (?, ?, ?)") + .bind("my_vault", userId, new Date().toISOString()) + .run(); + const expiresAt = new Date(Date.now() + 3_600_000).toISOString(); + await env.DB.batch([ + env.DB + .prepare( + "INSERT INTO tokens (jti, user_id, client_id, scopes, expires_at, revoked_at, created_at) VALUES (?, ?, ?, ?, ?, NULL, ?)", + ) + .bind("underscore-real", userId, "t", "vault:my_vault:read", expiresAt, expiresAt), + env.DB + .prepare( + "INSERT INTO tokens (jti, user_id, client_id, scopes, expires_at, revoked_at, created_at) VALUES (?, ?, ?, ?, ?, NULL, ?)", + ) + .bind("underscore-neighbor", userId, "t", "vault:myxvault:read", expiresAt, expiresAt), + ]); + const res = await handleAccountVaultDelete( db(), - accountReq("DELETE", "/account/vaults/doomed", { token }), + accountReq("DELETE", "/account/vaults/my_vault", { token, body: { confirm: "my_vault" } }), + accountDeps(), + ); + expect(res.status).toBe(200); + expect(((await res.json()) as { d1: { tokens_revoked: number } }).d1.tokens_revoked).toBe(1); + const neighbor = await env.DB.prepare("SELECT revoked_at FROM tokens WHERE jti = ?") + .bind("underscore-neighbor") + .first<{ revoked_at: string | null }>(); + expect(neighbor?.revoked_at).toBeNull(); + }); + + test("after the delete, the vault is gone from every read path — list, mint, and re-delete", async () => { + const { userId, token } = await seedOwnerWithPlan("del-after@example.com"); + await seedVault("erased", userId); + await seedVault("survivor", userId); + + const del = await handleAccountVaultDelete( + db(), + accountReq("DELETE", "/account/vaults/erased", { token, body: { confirm: "erased" } }), + accountDeps(), + ); + expect(del.status).toBe(200); + + // The list read path: the deleted vault is absent, its sibling untouched. + const list = await handleAccountVaultsList(db(), accountReq("GET", "/account/vaults", { token }), accountDeps()); + expect(list.status).toBe(200); + const listed = ((await list.json()) as { vaults: { name: string }[] }).vaults.map((v) => v.name); + expect(listed).toEqual(["survivor"]); + + // The mint path: no new token can be issued for the destroyed vault, and it + // refuses with the SAME neutral not_owner an unknown vault gets — the + // ownership row IS the gate, so removing it closes the mint by construction. + const mint = await handleAccountVaultTokenMint( + db(), + accountReq("POST", "/account/vaults/erased/token", { token, body: {} }), + accountDeps(), + ); + expect(mint.status).toBe(403); + expect(((await mint.json()) as { error: string }).error).toBe("not_owner"); + }); + + test("delete frees the D1 name claim and the plan's vault slot (NOT the same as the vault working again — see cloud#240)", async () => { + const { userId, token } = await seedOwnerWithPlan("del-reclaim@example.com"); + await seedVault("phoenix", userId); + + const del = await handleAccountVaultDelete( + db(), + accountReq("DELETE", "/account/vaults/phoenix", { token, body: { confirm: "phoenix" } }), + accountDeps(), + ); + expect(del.status).toBe(200); + + // Re-create: proves the ownership row really went (a lingering row would + // give `name_taken`) and that the vault-count cap counts the freed slot. + const recreated = await handleAccountVaultCreate( + db(), + accountReq("POST", "/account/vaults", { token, body: { name: "phoenix" } }), accountDeps(), ); - expect(res.status).toBe(501); - expect(((await res.json()) as { error: string }).error).toBe("not_implemented"); + expect(recreated.status).toBe(201); + const owner = await env.DB.prepare("SELECT owner_user_id FROM vaults WHERE name = ?") + .bind("phoenix") + .first<{ owner_user_id: string }>(); + expect(owner?.owner_user_id).toBe(userId); + + // SCOPE OF THIS TEST — read before trusting it, because its earlier title + // ("the same name can be created again") certified a property PRODUCTION + // DOES NOT HAVE. What is proven here is D1 bookkeeping ONLY: the name + // claim and the plan slot are released. Whether the RECREATED vault then + // works is a question about the Durable Object, and this suite cannot ask + // it — `accountDeps()` stubs `vaultFetch`, so no DO is involved at all. + // + // It does not work today. `idFromName` maps the reused name back to the + // SAME DO, whose in-memory `destroyed` latch (vault-do.ts) makes every + // subsequent request 410 `vault_destroyed` for as long as that instance + // stays resident — and each request keeps it resident. Filed as cloud#240; + // measured, not assumed (a probe confirmed the 410, and confirmed that + // deleteAll() drops every SQLite table, so the latch cannot simply be + // cleared: initSchema runs in the DO constructor). + // + // A test asserting the recreated vault is USABLE belongs in the vault + // worker's suite against a real DO, and would fail today. Asserting it + // here against a stub would only re-certify the same false property. + }); + + test("500 d1_cleanup_failed is honest when the sweep fails AFTER an irreversible destroy", async () => { + const { userId, token } = await seedOwnerWithPlan("del-d1-fail@example.com"); + await seedVault("half-torn", userId); + let destroyCalls = 0; + const vaultFetch: VaultFetch = async () => { + destroyCalls++; + return Response.json({ destroyed: true, r2_objects_deleted: 3 }, { status: 200 }); + }; + // Force the D1 batch to throw the only way the runtime lets us: remove a + // table the cascade writes. (Isolated per-test storage rolls this back.) + await env.DB.exec("DROP TABLE vault_snapshots"); + + const res = await handleAccountVaultDelete( + db(), + accountReq("DELETE", "/account/vaults/half-torn", { token, body: { confirm: "half-torn" } }), + accountDeps(undefined, vaultFetch), + ); + expect(res.status).toBe(500); + expect(((await res.json()) as { error: string }).error).toBe("d1_cleanup_failed"); + // Destroy DID run — the response must not pretend otherwise — and the whole + // request stays retryable: the ownership row is still there to retry from. + expect(destroyCalls).toBe(1); + const stillThere = await env.DB.prepare("SELECT 1 AS one FROM vaults WHERE name = ?") + .bind("half-torn") + .first<{ one: number }>(); + expect(stillThere).not.toBeNull(); + }); + + test("a second delete returns the same neutral 403 as an unknown vault and never errors", async () => { + const { userId, token } = await seedOwnerWithPlan("del-idempotent@example.com"); + await seedVault("gone", userId); + let calls = 0; + const vaultFetch: VaultFetch = async () => { + calls++; + return Response.json({ destroyed: true, r2_objects_deleted: 0 }, { status: 200 }); + }; + const first = await handleAccountVaultDelete( + db(), + accountReq("DELETE", "/account/vaults/gone", { token, body: { confirm: "gone" } }), + accountDeps(undefined, vaultFetch), + ); + expect(first.status).toBe(200); + + const second = await handleAccountVaultDelete( + db(), + accountReq("DELETE", "/account/vaults/gone", { token, body: { confirm: "gone" } }), + accountDeps(undefined, vaultFetch), + ); + expect(second.status).toBe(403); + expect(((await second.json()) as { error: string }).error).toBe("not_owner"); + expect(calls).toBe(1); }); test("401 for an unauthenticated DELETE (no shape leak)", async () => { - const res = await handleAccountVaultDelete(db(), accountReq("DELETE", "/account/vaults/x"), accountDeps()); + const res = await handleAccountVaultDelete( + db(), + accountReq("DELETE", "/account/vaults/x", { body: { confirm: "x" } }), + accountDeps(), + ); expect(res.status).toBe(401); }); - test("403 for a read token — the admin gate runs before the 501", async () => { + test("403 for a read token — the admin gate runs before confirmation or destroy", async () => { const { id } = await seedUser("del-read@example.com"); const readToken = await mintAccountToken(id, "read"); const res = await handleAccountVaultDelete( db(), - accountReq("DELETE", "/account/vaults/x", { token: readToken }), + accountReq("DELETE", "/account/vaults/x", { token: readToken, body: { confirm: "x" } }), accountDeps(), ); expect(res.status).toBe(403); }); + + test("400 confirm_mismatch is mutation-free", async () => { + const { userId, token } = await seedOwnerWithPlan("del-confirm@example.com"); + await seedVault("careful", userId); + let calls = 0; + const vaultFetch: VaultFetch = async () => { + calls++; + return Response.json({ destroyed: true, r2_objects_deleted: 0 }, { status: 200 }); + }; + const res = await handleAccountVaultDelete( + db(), + accountReq("DELETE", "/account/vaults/careful", { token, body: { confirm: "careless" } }), + accountDeps(undefined, vaultFetch), + ); + expect(res.status).toBe(400); + expect(((await res.json()) as { error: string }).error).toBe("confirm_mismatch"); + expect(calls).toBe(0); + expect((await deleteArtifactState("careful")).vaultExists).toBe(true); + }); + + test("403 for both another user's vault and an unknown vault, with no destroy call", async () => { + const a = await seedOwnerWithPlan("del-unowned-a@example.com"); + const b = await seedOwnerWithPlan("del-unowned-b@example.com"); + await seedVault("belongs-to-b", b.userId); + let calls = 0; + const vaultFetch: VaultFetch = async () => { + calls++; + return Response.json({ destroyed: true, r2_objects_deleted: 0 }, { status: 200 }); + }; + + for (const name of ["belongs-to-b", "does-not-exist"]) { + const res = await handleAccountVaultDelete( + db(), + accountReq("DELETE", `/account/vaults/${name}`, { token: a.token, body: { confirm: name } }), + accountDeps(undefined, vaultFetch), + ); + expect(res.status).toBe(403); + const body = (await res.json()) as { error: string; message: string }; + expect(body.error).toBe("not_owner"); + expect(Object.keys(body)).toEqual(["error", "message"]); + } + expect(calls).toBe(0); + }); + + test.each([ + ["non-2xx", async () => Response.json({ error: "busy" }, { status: 503 })], + ["transport", async () => { throw new Error("vault worker unreachable"); }], + // THE 200-THAT-ISN'T cases. Each of these is a plausible way the seam + // could answer 200 without the DO having destroyed anything: a renamed or + // shadowed route answering generically, a proxy/service-binding + // interposing, a future handler returning a different success shape. If + // any were believed, the D1 rows — the ONLY record of whose bytes those + // were — would go while the bytes stayed: orphaned storage, still billed, + // no longer attributable, unreachable by any retry. + ["200 with destroyed:false", async () => Response.json({ destroyed: false, r2_objects_deleted: 0 })], + ["200 with no destroyed field", async () => Response.json({ ok: true })], + ["200 with destroyed as a string", async () => Response.json({ destroyed: "true", r2_objects_deleted: 0 })], + ["200 with r2_objects_deleted missing", async () => Response.json({ destroyed: true })], + ["200 with r2_objects_deleted negative", async () => Response.json({ destroyed: true, r2_objects_deleted: -1 })], + ["200 with r2_objects_deleted fractional", async () => Response.json({ destroyed: true, r2_objects_deleted: 1.5 })], + ["200 with r2_objects_deleted as a string", async () => Response.json({ destroyed: true, r2_objects_deleted: "3" })], + ["200 with an unparseable body", async () => new Response("not json", { status: 200 })], + ] as const)("destroy %s leaves every D1 artifact intact", async (_kind, vaultFetch) => { + const { userId, token } = await seedOwnerWithPlan(`del-failure-${_kind.replace(/[^a-z0-9]+/gi, "-")}@example.com`); + await seedVault("failure-vault", userId); + const now = await seedDeleteArtifacts("failure-vault", userId); + const res = await handleAccountVaultDelete( + db(), + accountReq("DELETE", "/account/vaults/failure-vault", { token, body: { confirm: "failure-vault" } }), + accountDeps(() => now, vaultFetch), + ); + expect(res.status).toBe(502); + expect(((await res.json()) as { error: string }).error).toBe("vault_destroy_failed"); + expect(await deleteArtifactState("failure-vault")).toEqual({ + vaultExists: true, + usageRows: 2, + snapshotRows: 2, + liveTokenRevokedAt: null, + }); + }); }); // --- POST /account/vaults//token — per-vault mint ---------------------- diff --git a/workers/identity/test/account-delete.test.ts b/workers/identity/test/account-delete.test.ts new file mode 100644 index 0000000..978b2e0 --- /dev/null +++ b/workers/identity/test/account-delete.test.ts @@ -0,0 +1,1062 @@ +/** + * Account deletion end-to-end (cloud#226 A-3 + A-4, account-delete.ts) — the + * lifecycle Aaron ratified, one describe per leg: + * + * 1. REQUEST — auth severed immediately, billing held reversibly, the + * tombstone + undo hash written, and NOTHING erased. + * 2. THE WINDOW — for 24 hours the data is intact but unreachable, and the + * sweep will not touch it. + * 3. UNDO — inside the window the account comes back, billing is released, + * the token is single-use, and every unusable token gets one neutral + * answer. + * 4. CONVERGENCE — past the window the sweep really removes the billing + * artifacts AND the vault storage AND the account rows, and afterwards + * every read refuses. + * + * WHAT IS NOT RE-PROVEN HERE. The Stripe mechanics of deferBilling / + * resumeBilling / teardownBilling (tolerable-vs-hard errors, the cloud#64 + * orphan belt, the NULL-ids converged marker) are billing-teardown.test.ts's + * subject and are pinned there against the same injected-stub seam; this suite + * asserts only that the ROUTES call them at the right moment and honor their + * results. Likewise the read-time refusal chokepoints are A-1's + * (account-api/auth/console/drip/usage/snapshot tests) — what is proven here is + * that this route is the thing that finally SETS the column they read. + * + * The vault teardown is driven through the injected `vaultFetch` seam (no DO is + * reachable in-test), so what these tests pin about storage is the CONTRACT: + * that destroy is called, once per owned vault, with the confirm body and an + * admin mint for that vault — and, critically, that the D1 ownership rows are + * removed only when it succeeded. + */ +import { env } from "cloudflare:test"; +import type Stripe from "stripe"; +import { describe, expect, test } from "vitest"; +import { ACCOUNT_TOKEN_AUDIENCE } from "../src/account-auth.ts"; +import { handleAccountVaultsList } from "../src/account-api.ts"; +import { + ACCOUNT_PURGE_ALERT_AFTER_ATTEMPTS, + DELETE_UNDO_WINDOW_MS, + handleAccountDelete, + handleAccountDeleteUndo, + runAccountDeleteSweep, +} from "../src/account-delete.ts"; +import { ACCOUNT_TOKEN_CLIENT_ID } from "../src/account-token.ts"; +import type { BillingOverrides } from "../src/billing.ts"; +import { sha256Hex } from "../src/crypto.ts"; +import type { EmailSender, OpsEmail, SendResult } from "../src/email.ts"; +import type { OAuthDeps } from "../src/oauth-shared.ts"; +import { eligibleFor } from "../src/drip.ts"; +import { DRIP_CRON, handleScheduled } from "../src/ops.ts"; +import { findActiveSession } from "../src/sessions.ts"; +import { signAccessToken } from "../src/tokens.ts"; +import { getUserById } from "../src/users.ts"; +import { ISSUER, db, decodeJwtPayload, deps, seedSession, seedUser, seedVault } from "./helpers.ts"; + +type VaultFetch = NonNullable; + +/** A destroy call the vault worker would have received. */ +interface DestroyCall { + method: string; + path: string; + body: unknown; + scope: string; + audience: unknown; +} + +/** The default vault seam: every destroy succeeds, and each call is recorded so + * a test can assert the contract (one call per owned vault, confirm body, + * vault-scoped admin mint) instead of merely that "something happened". */ +function recordingVaultFetch(opts: { failFor?: Set } = {}): { + vaultFetch: VaultFetch; + calls: DestroyCall[]; +} { + const calls: DestroyCall[] = []; + const vaultFetch: VaultFetch = async (input, init) => { + const req = new Request(input, init); + const path = new URL(req.url).pathname; + const body = await req.clone().json().catch(() => null); + const token = req.headers.get("authorization")?.replace(/^Bearer\s+/, "") ?? ""; + const claims = token ? decodeJwtPayload(token) : {}; + calls.push({ + method: req.method, + path, + body, + scope: String(claims.scope ?? ""), + audience: claims.aud, + }); + const name = String((body as { confirm?: string } | null)?.confirm ?? ""); + if (opts.failFor?.has(name)) return Response.json({ error: "busy" }, { status: 503 }); + return Response.json({ destroyed: true, r2_objects_deleted: 1 }, { status: 200 }); + }; + return { vaultFetch, calls }; +} + +function accountDeps(now?: () => Date, vaultFetch?: VaultFetch): OAuthDeps { + return { ...deps(now), vaultFetch: vaultFetch ?? recordingVaultFetch().vaultFetch }; +} + +async function mintAccountToken(userId: string, verb: "admin" | "read" = "admin"): Promise { + const signed = await signAccessToken(db(), { + sub: userId, + scopes: [`account:${userId}:${verb}`], + audience: ACCOUNT_TOKEN_AUDIENCE, + clientId: ACCOUNT_TOKEN_CLIENT_ID, + issuer: ISSUER, + vaultScope: [], + ttlSeconds: 600, + }); + return signed.token; +} + +function deleteReq(token: string | null, confirm?: unknown, method: "DELETE" | "POST" = "DELETE"): Request { + const headers: Record = { "content-type": "application/json" }; + if (token) headers.authorization = `Bearer ${token}`; + return new Request(`${ISSUER}/account/delete`, { + method, + headers, + body: JSON.stringify(confirm === undefined ? {} : { confirm }), + }); +} + +function undoGetReq(token: string): Request { + return new Request(`${ISSUER}/account/undo-delete?token=${encodeURIComponent(token)}`); +} + +function undoPostReq(body: unknown): Request { + return new Request(`${ISSUER}/account/undo-delete`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }); +} + +// --- the injected Stripe stub (billing-teardown.test.ts's seam) --------------- + +interface StripeCalls { + updated: Array<{ id: string; cancelAtPeriodEnd: boolean }>; + canceled: string[]; + deletedCustomers: string[]; +} + +function makeStripeStub(opts: { hardFailUpdate?: boolean } = {}): { stripe: Stripe; calls: StripeCalls } { + const calls: StripeCalls = { updated: [], canceled: [], deletedCustomers: [] }; + const stripe = { + subscriptions: { + update: async (id: string, params: { cancel_at_period_end?: boolean }) => { + calls.updated.push({ id, cancelAtPeriodEnd: params.cancel_at_period_end ?? false }); + if (opts.hardFailUpdate) throw new Error("stripe is down"); + return { id, status: "active" }; + }, + cancel: async (id: string) => { + calls.canceled.push(id); + return { id, status: "canceled" }; + }, + list: async () => ({ object: "list", data: [], has_more: false, url: "/v1/subscriptions" }), + }, + customers: { + del: async (id: string) => { + calls.deletedCustomers.push(id); + return { id, object: "customer", deleted: true }; + }, + }, + } as unknown as Stripe; + return { stripe, calls }; +} + +function billing(stripe: Stripe): BillingOverrides { + return { stripe }; +} + +/** An EmailSender that records the deletion notice instead of sending it. */ +function recordingSender(opts: { fail?: boolean } = {}): { sender: EmailSender; ops: OpsEmail[] } { + const ops: OpsEmail[] = []; + const sender: EmailSender = { + kind: "devlog", + async sendMagicLink(): Promise { + return { ok: true }; + }, + async sendOps(msg: OpsEmail): Promise { + ops.push(msg); + return opts.fail ? { ok: false, error: "mailbox full" } : { ok: true }; + }, + async sendDrip(): Promise { + return { ok: true }; + }, + }; + return { sender, ops }; +} + +// --- fixtures ----------------------------------------------------------------- + +const T0 = new Date("2026-08-10T00:00:00.000Z"); +const clock = (d: Date) => () => d; +const plus = (ms: number) => new Date(T0.getTime() + ms); + +interface Seeded { + userId: string; + email: string; + token: string; + sessionId: string; +} + +/** A full account: bearer, live session, a live vault token, a consent grant, + * and (optionally) Stripe ids — i.e. one of every artifact the teardown owns. */ +async function seedAccount( + email: string, + opts: { vaults?: string[]; stripe?: boolean; handle?: string } = {}, +): Promise { + const { id } = await seedUser(email); + const token = await mintAccountToken(id); + const sessionId = await seedSession(id); + for (const name of opts.vaults ?? []) await seedVault(name, id); + + const expiresAt = new Date(T0.getTime() + 3_600_000).toISOString(); + await env.DB.batch([ + env.DB + .prepare( + "INSERT INTO tokens (jti, user_id, client_id, scopes, expires_at, revoked_at, created_at) VALUES (?, ?, ?, ?, ?, NULL, ?)", + ) + .bind(`jti-${id}`, id, "some-client", `vault:${(opts.vaults ?? ["none"])[0]}:read`, expiresAt, T0.toISOString()), + env.DB + .prepare("INSERT INTO grants (user_id, client_id, scopes, granted_at) VALUES (?, ?, ?, ?)") + .bind(id, "some-client", `vault:${(opts.vaults ?? ["none"])[0]}:read`, T0.toISOString()), + env.DB + .prepare("INSERT INTO user_checklist (user_id, item, done_at) VALUES (?, ?, ?)") + .bind(id, "created-vault", T0.toISOString()), + ]); + + if (opts.stripe) { + await env.DB + .prepare("UPDATE users SET stripe_customer_id = ?, stripe_subscription_id = ?, plan = 'standard' WHERE id = ?") + .bind(`cus_${id}`, `sub_${id}`, id) + .run(); + } + if (opts.handle) { + await env.DB + .prepare("INSERT INTO owners (owner_id, handle, kind, claimed_at) VALUES (?, ?, 'user', ?)") + .bind(`own_${id}`, opts.handle, T0.toISOString()) + .run(); + await env.DB.prepare("UPDATE users SET owner_id = ? WHERE id = ?").bind(`own_${id}`, id).run(); + } + return { userId: id, email, token, sessionId }; +} + +async function countRows(table: string, column: string, value: string): Promise { + const row = await env.DB + .prepare(`SELECT COUNT(*) AS n FROM ${table} WHERE ${column} = ?`) + .bind(value) + .first<{ n: number }>(); + return row?.n ?? 0; +} + +/** Everything the teardown is supposed to remove, in one shot. */ +async function residue(userId: string, vaultName: string) { + return { + user: (await getUserById(env.DB, userId)) !== null, + vaults: await countRows("vaults", "owner_user_id", userId), + vaultUsage: await countRows("vault_usage", "vault_name", vaultName), + vaultSnapshots: await countRows("vault_snapshots", "vault_name", vaultName), + sessions: await countRows("sessions", "user_id", userId), + tokens: await countRows("tokens", "user_id", userId), + grants: await countRows("grants", "user_id", userId), + checklist: await countRows("user_checklist", "user_id", userId), + }; +} + +/** The per-vault rollup/manifest rows the vault delete cascade owns. */ +async function seedVaultMirrors(vaultName: string): Promise { + await env.DB.batch([ + env.DB + .prepare("INSERT INTO vault_usage (vault_name, day, db_bytes, r2_bytes) VALUES (?, ?, ?, ?)") + .bind(vaultName, "2026-08-09", 11, 22), + env.DB + .prepare("INSERT INTO vault_snapshots (vault_name, key, taken_at, bytes, ranks) VALUES (?, ?, ?, ?, ?)") + .bind(vaultName, `vault-${vaultName}/snapshots/a.tar`, T0.toISOString(), 33, "[]"), + ]); +} + +// --- 1. the request ---------------------------------------------------------- + +describe("A-3 — DELETE /account/delete opens the undo window", () => { + test("tombstones the account, holds billing reversibly, severs auth, and erases NOTHING", async () => { + const acct = await seedAccount("req-happy@example.com", { vaults: ["keepsafe"], stripe: true }); + await seedVaultMirrors("keepsafe"); + const { stripe, calls } = makeStripeStub(); + const { sender, ops } = recordingSender(); + + const res = await handleAccountDelete( + env, + deleteReq(acct.token, acct.email), + accountDeps(clock(T0)), + sender, + billing(stripe), + ); + expect(res.status).toBe(200); + const body = (await res.json()) as Record; + expect(body.deleted).toBe(true); + expect(body.deleted_at).toBe(T0.toISOString()); + expect(body.undo_expires_at).toBe(new Date(T0.getTime() + DELETE_UNDO_WINDOW_MS).toISOString()); + expect(body.notice_sent).toBe(true); + expect(typeof body.undo_token).toBe("string"); + + // The tombstone is real, and the stored undo secret is a HASH — the raw + // token must never be at rest (it is mailed, and it restores an account). + const user = await getUserById(env.DB, acct.userId); + expect(user?.deletedAt).toBe(T0.toISOString()); + expect(user?.deleteNoticeSentAt).toBe(T0.toISOString()); + expect(user?.deleteUndoHash).toBe(await sha256Hex(String(body.undo_token))); + expect(user?.deleteUndoHash).not.toBe(String(body.undo_token)); + + // The hold is the REVERSIBLE one (cancel_at_period_end), never a cancel — + // this is the whole point of the window. Goes red if the route ever calls + // teardownBilling at request time. + expect(calls.updated).toEqual([{ id: `sub_${acct.userId}`, cancelAtPeriodEnd: true }]); + expect(calls.canceled).toEqual([]); + expect(calls.deletedCustomers).toEqual([]); + + // Auth is severed NOW, not at the window's end. + expect(await findActiveSession(env.DB, acct.sessionId, T0)).toBeNull(); + const live = await countRows("tokens", "user_id", acct.userId); + const revoked = await env.DB + .prepare("SELECT COUNT(*) AS n FROM tokens WHERE user_id = ? AND revoked_at IS NOT NULL") + .bind(acct.userId) + .first<{ n: number }>(); + expect(revoked?.n).toBe(live); + + // …and NOTHING has been erased. The vault, its mirrors, and the account + // row all survive the request — only the sweep may remove them. + expect(await residue(acct.userId, "keepsafe")).toMatchObject({ + user: true, + vaults: 1, + vaultUsage: 1, + vaultSnapshots: 1, + }); + + // The notice carries the way back — a user who was just signed out + // everywhere has no other route to undo. + expect(ops).toHaveLength(1); + expect(ops[0]!.to).toBe(acct.email); + expect(ops[0]!.text).toContain(String(body.undo_url)); + }); + + test("the severed bearer is dead on the very next call (A-1's chokepoint, now reachable)", async () => { + const acct = await seedAccount("req-severed@example.com"); + const first = await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + expect(first.status).toBe(200); + + // The SAME bearer that just worked now gets the "account not found" shape — + // indistinguishable from a missing row, by A-1's design. + const second = await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + expect(second.status).toBe(401); + expect((await second.json()) as unknown).toEqual({ + error: "invalid_token", + error_description: "account not found", + }); + }); + + test("confirm must retype the account email, and a mismatch changes nothing", async () => { + const acct = await seedAccount("req-confirm@example.com", { stripe: true }); + const { stripe, calls } = makeStripeStub(); + for (const bad of [undefined, "", "someone-else@example.com", 42]) { + const res = await handleAccountDelete( + env, + deleteReq(acct.token, bad), + accountDeps(clock(T0)), + undefined, + billing(stripe), + ); + expect(res.status).toBe(400); + expect(((await res.json()) as { error: string }).error).toBe("confirm_mismatch"); + } + // Not one Stripe call, not one row touched — the confirm gate runs first. + expect(calls.updated).toEqual([]); + expect((await getUserById(env.DB, acct.userId))?.deletedAt).toBeNull(); + expect(await findActiveSession(env.DB, acct.sessionId, T0)).not.toBeNull(); + }); + + test("case-insensitive confirm (the email column is NOCASE — the gate must agree)", async () => { + const acct = await seedAccount("Req-Case@example.com"); + const res = await handleAccountDelete( + env, + deleteReq(acct.token, " REQ-CASE@EXAMPLE.COM "), + accountDeps(clock(T0)), + ); + expect(res.status).toBe(200); + }); + + test("401 unauthenticated and 403 for a read-scope bearer — both before any destruction", async () => { + const { id } = await seedUser("req-scope@example.com"); + const readToken = await mintAccountToken(id, "read"); + + const anon = await handleAccountDelete(env, deleteReq(null, "req-scope@example.com"), accountDeps(clock(T0))); + expect(anon.status).toBe(401); + + const read = await handleAccountDelete(env, deleteReq(readToken, "req-scope@example.com"), accountDeps(clock(T0))); + expect(read.status).toBe(403); + expect((await getUserById(env.DB, id))?.deletedAt).toBeNull(); + }); + + test("a hard billing failure aborts the request with nothing written", async () => { + const acct = await seedAccount("req-billing-down@example.com", { stripe: true }); + const { stripe } = makeStripeStub({ hardFailUpdate: true }); + + const res = await handleAccountDelete( + env, + deleteReq(acct.token, acct.email), + accountDeps(clock(T0)), + undefined, + billing(stripe), + ); + expect(res.status).toBe(502); + expect(((await res.json()) as { error: string }).error).toBe("billing_hold_failed"); + // The hold is the FIRST step precisely so this case leaves no tombstone: + // an account that still bills must not also be un-loggable-into. + expect((await getUserById(env.DB, acct.userId))?.deletedAt).toBeNull(); + expect(await findActiveSession(env.DB, acct.sessionId, T0)).not.toBeNull(); + }); + + test("a hold that is placed and then cannot be released pages the operator", async () => { + // The compound failure: deferBilling succeeds, the tombstone write fails, + // and resumeBilling ALSO fails. The subscription is left at + // cancel_at_period_end on an account with NO tombstone — so no sweep will + // ever revisit it and the user's subscription silently lapses. Nothing in + // the product shows this; the alert is the only way it becomes work. + const acct = await seedAccount("req-hold-stuck@example.com", { stripe: true }); + let updateCalls = 0; + const oneWayStripe = { + subscriptions: { + update: async (_id: string, params: { cancel_at_period_end?: boolean }) => { + updateCalls++; + // The hold lands; the RELEASE is what fails. + if (params.cancel_at_period_end === false) throw new Error("stripe is down"); + return { id: _id, status: "active" }; + }, + }, + } as unknown as Stripe; + const { sender, ops } = recordingSender(); + const alertEnv = { ...env, OPERATOR_ALERT_EMAIL: "ops@example.com", ENVIRONMENT: "test-env" }; + + // Fail the TOMBSTONE WRITE SPECIFICALLY. Dropping `users` would be caught + // earlier, by requireAccount's own read, and never reach the code under + // test — so block exactly the one UPDATE instead, with a trigger. (Isolated + // per-test storage rolls this back.) + await env.DB + .prepare( + "CREATE TRIGGER block_tombstone BEFORE UPDATE OF deleted_at ON users WHEN NEW.deleted_at IS NOT NULL BEGIN SELECT RAISE(ABORT, 'tombstone blocked'); END", + ) + .run(); + + const res = await handleAccountDelete( + alertEnv, + deleteReq(acct.token, acct.email), + accountDeps(clock(T0)), + sender, + billing(oneWayStripe), + ); + expect(res.status).toBe(500); + expect(((await res.json()) as { error: string }).error).toBe("delete_request_failed"); + // Hold placed, release attempted and failed. + expect(updateCalls).toBe(2); + + const alert = ops.find((m) => m.subject.includes("billing hold stuck")); + expect(alert).toBeDefined(); + expect(alert!.to).toBe("ops@example.com"); + expect(alert!.text).toContain(acct.userId); + expect(alert!.text).toContain(`sub_${acct.userId}`); + // The operator needs to know this account is NOT deleted — that is what + // makes it unreachable by every automatic path. + expect(alert!.text).toContain("NOT deleted"); + }); + + test("a failed notice email does not fail the deletion, and never claims it was sent", async () => { + const acct = await seedAccount("req-mail-down@example.com"); + const { sender, ops } = recordingSender({ fail: true }); + const res = await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0)), sender); + expect(res.status).toBe(200); + expect(((await res.json()) as { notice_sent: boolean }).notice_sent).toBe(false); + expect(ops).toHaveLength(1); + // The delete stands (it is already in effect), but the column stays NULL — + // it must never assert an email that did not go out. + const user = await getUserById(env.DB, acct.userId); + expect(user?.deletedAt).toBe(T0.toISOString()); + expect(user?.deleteNoticeSentAt).toBeNull(); + }); +}); + +// --- 2. the window ----------------------------------------------------------- + +describe("A-4 — inside the 24-hour window nothing is destroyed", () => { + test("a sweep one second before expiry finds nothing due and leaves every artifact", async () => { + const acct = await seedAccount("win-early@example.com", { vaults: ["patient"], stripe: true }); + await seedVaultMirrors("patient"); + const { stripe, calls } = makeStripeStub(); + await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0)), undefined, billing(stripe)); + + const { vaultFetch, calls: destroys } = recordingVaultFetch(); + const summary = await runAccountDeleteSweep( + env, + accountDeps(undefined, vaultFetch), + plus(DELETE_UNDO_WINDOW_MS - 1000), + billing(stripe), + ); + expect(summary).toEqual({ due: 0, purged: 0, deferred: 0, failed: 0 }); + expect(destroys).toEqual([]); + expect(calls.canceled).toEqual([]); + expect(await residue(acct.userId, "patient")).toMatchObject({ + user: true, + vaults: 1, + vaultUsage: 1, + vaultSnapshots: 1, + }); + }); +}); + +// --- 3. undo ----------------------------------------------------------------- + +describe("A-3 — undo inside the window restores the account", () => { + test("the emailed GET link restores the account and releases the billing hold", async () => { + const acct = await seedAccount("undo-link@example.com", { vaults: ["restored"], stripe: true }); + const { stripe, calls } = makeStripeStub(); + const del = await handleAccountDelete( + env, + deleteReq(acct.token, acct.email), + accountDeps(clock(T0)), + undefined, + billing(stripe), + ); + const { undo_token } = (await del.json()) as { undo_token: string }; + + const res = await handleAccountDeleteUndo( + env, + undoGetReq(undo_token), + accountDeps(clock(plus(60 * 60 * 1000))), + billing(stripe), + ); + expect(res.status).toBe(200); + expect(await res.json()).toEqual({ restored: true, billing_resumed: true }); + + // The tombstone AND the spent secret are both cleared — a restored account + // must not keep a live undo token lying around. + const user = await getUserById(env.DB, acct.userId); + expect(user?.deletedAt).toBeNull(); + expect(user?.deleteUndoHash).toBeNull(); + expect(user?.deleteNoticeSentAt).toBeNull(); + + // The hold was released, not re-applied, and nothing was ever canceled. + expect(calls.updated).toEqual([ + { id: `sub_${acct.userId}`, cancelAtPeriodEnd: true }, + { id: `sub_${acct.userId}`, cancelAtPeriodEnd: false }, + ]); + expect(calls.canceled).toEqual([]); + + // And the account works again: a fresh bearer passes the gate that refused + // it a moment ago, and still owns its vault. + const fresh = await mintAccountToken(acct.userId); + const list = await handleAccountVaultsList( + db(), + new Request(`${ISSUER}/account/vaults`, { headers: { authorization: `Bearer ${fresh}` } }), + accountDeps(), + ); + expect(list.status).toBe(200); + expect(((await list.json()) as { vaults: { name: string }[] }).vaults.map((v) => v.name)).toEqual(["restored"]); + }); + + test("POST with a JSON body is the same door (API clients have no inbox)", async () => { + const acct = await seedAccount("undo-post@example.com"); + const del = await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + const { undo_token } = (await del.json()) as { undo_token: string }; + + const res = await handleAccountDeleteUndo(env, undoPostReq({ token: undo_token }), accountDeps(clock(plus(1000)))); + expect(res.status).toBe(200); + expect((await getUserById(env.DB, acct.userId))?.deletedAt).toBeNull(); + }); + + test("an un-resumable subscription restores the account but says so plainly", async () => { + const acct = await seedAccount("undo-uncancelable@example.com", { stripe: true }); + // deferBilling succeeds; the period boundary then passes, so the resume + // update hits Stripe's tolerable invalid-request shape — resumeBilling's + // `already_canceled`. Swapped in AFTER the delete so only the undo sees it. + const { stripe: holdStripe } = makeStripeStub(); + const del = await handleAccountDelete( + env, + deleteReq(acct.token, acct.email), + accountDeps(clock(T0)), + undefined, + billing(holdStripe), + ); + const { undo_token } = (await del.json()) as { undo_token: string }; + + const StripeCtor = (await import("stripe")).default; + const canceledStripe = { + subscriptions: { + update: async () => { + throw new StripeCtor.errors.StripeInvalidRequestError({ + message: "cannot update a canceled subscription", + statusCode: 400, + }); + }, + }, + } as unknown as Stripe; + + const res = await handleAccountDeleteUndo( + env, + undoPostReq({ token: undo_token }), + accountDeps(clock(plus(1000))), + billing(canceledStripe), + ); + expect(res.status).toBe(200); + // The account comes back — the data is theirs regardless — but the caller + // is told billing did not, instead of discovering it on a failed feature. + expect(await res.json()).toEqual({ restored: true, billing_resumed: false, billing_note: "already_canceled" }); + expect((await getUserById(env.DB, acct.userId))?.deletedAt).toBeNull(); + }); + + test("a hard billing failure leaves the account deleted and the window open", async () => { + const acct = await seedAccount("undo-billing-down@example.com", { stripe: true }); + const { stripe: holdStripe } = makeStripeStub(); + const del = await handleAccountDelete( + env, + deleteReq(acct.token, acct.email), + accountDeps(clock(T0)), + undefined, + billing(holdStripe), + ); + const { undo_token } = (await del.json()) as { undo_token: string }; + + const { stripe: brokenStripe } = makeStripeStub({ hardFailUpdate: true }); + const res = await handleAccountDeleteUndo( + env, + undoPostReq({ token: undo_token }), + accountDeps(clock(plus(1000))), + billing(brokenStripe), + ); + expect(res.status).toBe(502); + expect(((await res.json()) as { error: string }).error).toBe("billing_resume_failed"); + // Still deleted, and the token still works once Stripe recovers — a + // half-restored account (auth back, billing unknown) is the bad outcome. + expect((await getUserById(env.DB, acct.userId))?.deletedAt).toBe(T0.toISOString()); + const retry = await handleAccountDeleteUndo( + env, + undoPostReq({ token: undo_token }), + accountDeps(clock(plus(2000))), + billing(makeStripeStub().stripe), + ); + expect(retry.status).toBe(200); + }); + + test("the undo token is single-use", async () => { + const acct = await seedAccount("undo-once@example.com"); + const del = await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + const { undo_token } = (await del.json()) as { undo_token: string }; + + expect((await handleAccountDeleteUndo(env, undoGetReq(undo_token), accountDeps(clock(plus(1000))))).status).toBe(200); + const second = await handleAccountDeleteUndo(env, undoGetReq(undo_token), accountDeps(clock(plus(2000)))); + expect(second.status).toBe(400); + expect(((await second.json()) as { error: string }).error).toBe("invalid_undo_token"); + }); + + test("every unusable token gets ONE neutral answer — no oracle", async () => { + const acct = await seedAccount("undo-neutral@example.com"); + // A live account's own id, a random string, an empty token, and a + // never-issued one must be indistinguishable from each other. + const bodies: unknown[] = []; + for (const req of [ + undoPostReq({}), + undoPostReq({ token: "" }), + undoPostReq({ token: "not-a-real-token" }), + undoPostReq({ token: acct.userId }), + undoGetReq("nope"), + ]) { + const res = await handleAccountDeleteUndo(env, req, accountDeps(clock(T0))); + expect(res.status).toBe(400); + bodies.push(await res.json()); + } + expect(new Set(bodies.map((b) => JSON.stringify(b))).size).toBe(1); + }); + + test("past the window the token is refused with the honest 410, not the neutral 400", async () => { + const acct = await seedAccount("undo-late@example.com"); + const del = await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + const { undo_token } = (await del.json()) as { undo_token: string }; + + const res = await handleAccountDeleteUndo( + env, + undoGetReq(undo_token), + accountDeps(clock(plus(DELETE_UNDO_WINDOW_MS))), + ); + expect(res.status).toBe(410); + expect(((await res.json()) as { error: string }).error).toBe("undo_window_expired"); + // Deliberately NOT neutral: this token was real, the holder is the account + // holder, and "you're too late" is the only useful thing to tell them. + expect((await getUserById(env.DB, acct.userId))?.deletedAt).toBe(T0.toISOString()); + }); + + test("the boundary is exclusive at exactly 24h — one millisecond earlier still restores", async () => { + const acct = await seedAccount("undo-boundary@example.com"); + const del = await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + const { undo_token } = (await del.json()) as { undo_token: string }; + const res = await handleAccountDeleteUndo( + env, + undoGetReq(undo_token), + accountDeps(clock(plus(DELETE_UNDO_WINDOW_MS - 1))), + ); + expect(res.status).toBe(200); + }); +}); + +// --- 4. convergence ---------------------------------------------------------- + +describe("A-4 — past the window the sweep really deletes", () => { + test("tears down billing, destroys every owned vault, and purges the account", async () => { + const acct = await seedAccount("purge-happy@example.com", { + vaults: ["alpha", "beta"], + stripe: true, + handle: "purged-handle", + }); + await seedVaultMirrors("alpha"); + await seedVaultMirrors("beta"); + const { stripe, calls } = makeStripeStub(); + await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0)), undefined, billing(stripe)); + + const { vaultFetch, calls: destroys } = recordingVaultFetch(); + const summary = await runAccountDeleteSweep( + env, + accountDeps(undefined, vaultFetch), + plus(DELETE_UNDO_WINDOW_MS), + billing(stripe), + ); + expect(summary).toEqual({ due: 1, purged: 1, deferred: 0, failed: 0 }); + + // STORAGE: one destroy per owned vault, each an admin mint pinned to that + // vault with the confirm body — the same contract the single-vault door + // honors, because it is literally the same call. + expect(destroys).toHaveLength(2); + for (const name of ["alpha", "beta"]) { + const call = destroys.find((c) => (c.body as { confirm: string }).confirm === name); + expect(call).toBeDefined(); + expect(call!.method).toBe("POST"); + expect(call!.path).toContain("/api/internal/destroy"); + expect(call!.scope).toBe(`vault:${name}:admin`); + expect(call!.audience).toBe(`vault.${name}`); + } + + // BILLING: the real, irreversible teardown ran — cancel + delete-customer, + // which the request path deliberately did NOT do. + expect(calls.canceled).toEqual([`sub_${acct.userId}`]); + expect(calls.deletedCustomers).toEqual([`cus_${acct.userId}`]); + + // ROWS: nothing of the account is left, in any table it touched. + expect(await residue(acct.userId, "alpha")).toEqual({ + user: false, + vaults: 0, + vaultUsage: 0, + vaultSnapshots: 0, + sessions: 0, + tokens: 0, + grants: 0, + checklist: 0, + }); + expect(await countRows("vault_usage", "vault_name", "beta")).toBe(0); + // The handle is RELEASED, not burned — a global namespace claim must not + // outlive the account that made it. + expect(await countRows("owners", "handle", "purged-handle")).toBe(0); + }); + + test("after the purge every read refuses — the bearer, the vault list, and the undo token", async () => { + const acct = await seedAccount("purge-reads@example.com", { vaults: ["vanished"] }); + const del = await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + const { undo_token } = (await del.json()) as { undo_token: string }; + const fresh = await mintAccountToken(acct.userId); + + await runAccountDeleteSweep(env, accountDeps(), plus(DELETE_UNDO_WINDOW_MS)); + + // A bearer minted while the account existed is now indistinguishable from + // one for an account that never existed. + const list = await handleAccountVaultsList( + db(), + new Request(`${ISSUER}/account/vaults`, { headers: { authorization: `Bearer ${fresh}` } }), + accountDeps(), + ); + expect(list.status).toBe(401); + expect((await list.json()) as unknown).toEqual({ + error: "invalid_token", + error_description: "account not found", + }); + + // The undo token is dead too — and gets the expired answer, not a 500 on a + // row that no longer exists. + const undo = await handleAccountDeleteUndo( + env, + undoGetReq(undo_token), + accountDeps(clock(plus(DELETE_UNDO_WINDOW_MS + 1000))), + ); + expect(undo.status).toBe(400); + expect(await countRows("vaults", "name", "vanished")).toBe(0); + }); + + test("a vault whose destroy FAILS defers the whole account, and a later pass converges", async () => { + const acct = await seedAccount("purge-stuck@example.com", { vaults: ["wedged", "fine"] }); + await seedVaultMirrors("wedged"); + await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + + const failing = recordingVaultFetch({ failFor: new Set(["wedged"]) }); + const first = await runAccountDeleteSweep( + env, + accountDeps(undefined, failing.vaultFetch), + plus(DELETE_UNDO_WINDOW_MS), + ); + expect(first).toEqual({ due: 1, purged: 0, deferred: 1, failed: 0 }); + + // The account row SURVIVES — it is the only thing that still says whose + // the surviving vault is. The vault that did succeed is gone (destroy is + // irreversible, so its rows must not be re-created), the wedged one stays. + expect((await getUserById(env.DB, acct.userId))?.deletedAt).toBe(T0.toISOString()); + expect(await countRows("vaults", "name", "wedged")).toBe(1); + expect(await countRows("vault_usage", "vault_name", "wedged")).toBe(1); + expect(await countRows("vaults", "name", "fine")).toBe(0); + + // Next pass, DO healthy: converges, and does not re-destroy the vault that + // already went (its ownership row is gone, so it is not enumerated). + const healthy = recordingVaultFetch(); + const second = await runAccountDeleteSweep( + env, + accountDeps(undefined, healthy.vaultFetch), + plus(DELETE_UNDO_WINDOW_MS + 3_600_000), + ); + expect(second).toEqual({ due: 1, purged: 1, deferred: 0, failed: 0 }); + expect(healthy.calls.map((c) => (c.body as { confirm: string }).confirm)).toEqual(["wedged"]); + expect(await residue(acct.userId, "wedged")).toMatchObject({ user: false, vaults: 0, vaultUsage: 0 }); + }); + + test("billing that will not converge defers the account and never touches vault storage", async () => { + const acct = await seedAccount("purge-billing-stuck@example.com", { vaults: ["untouched"], stripe: true }); + const { stripe: holdStripe } = makeStripeStub(); + await handleAccountDelete( + env, + deleteReq(acct.token, acct.email), + accountDeps(clock(T0)), + undefined, + billing(holdStripe), + ); + + // teardownBilling catches its own failures and answers unconverged. + const brokenStripe = { + subscriptions: { + cancel: async () => { + throw new Error("stripe is down"); + }, + list: async () => ({ object: "list", data: [], has_more: false, url: "/v1/subscriptions" }), + }, + customers: { del: async () => ({ deleted: true }) }, + } as unknown as Stripe; + + const { vaultFetch, calls: destroys } = recordingVaultFetch(); + const summary = await runAccountDeleteSweep( + env, + accountDeps(undefined, vaultFetch), + plus(DELETE_UNDO_WINDOW_MS), + billing(brokenStripe), + ); + expect(summary).toEqual({ due: 1, purged: 0, deferred: 1, failed: 0 }); + // Money first, and it stops the pass: destroying the data while a + // subscription may still be billing for it is the worst ordering. + expect(destroys).toEqual([]); + expect(await countRows("vaults", "name", "untouched")).toBe(1); + expect((await getUserById(env.DB, acct.userId))?.deletedAt).toBe(T0.toISOString()); + }); + + test("one wedged account does not block the rest of the queue", async () => { + const stuck = await seedAccount("purge-iso-stuck@example.com", { vaults: ["stuckvault"] }); + const ok = await seedAccount("purge-iso-ok@example.com", { vaults: ["okvault"] }); + await handleAccountDelete(env, deleteReq(stuck.token, stuck.email), accountDeps(clock(T0))); + await handleAccountDelete(env, deleteReq(ok.token, ok.email), accountDeps(clock(new Date(T0.getTime() + 1000)))); + + const { vaultFetch } = recordingVaultFetch({ failFor: new Set(["stuckvault"]) }); + const summary = await runAccountDeleteSweep( + env, + accountDeps(undefined, vaultFetch), + plus(DELETE_UNDO_WINDOW_MS + 2000), + ); + expect(summary).toEqual({ due: 2, purged: 1, deferred: 1, failed: 0 }); + expect((await getUserById(env.DB, stuck.userId)) !== null).toBe(true); + expect((await getUserById(env.DB, ok.userId)) === null).toBe(true); + }); + + test.each([ + ["destroyed:false", { destroyed: false, r2_objects_deleted: 0 }], + ["no destroyed field", { ok: true }], + ["r2_objects_deleted missing", { destroyed: true }], + ["r2_objects_deleted negative", { destroyed: true, r2_objects_deleted: -1 }], + ["r2_objects_deleted fractional", { destroyed: true, r2_objects_deleted: 2.5 }], + ] as const)( + "a 200 that is not the DO's real reply (%s) defers instead of deleting the rows", + async (kind, body) => { + // THE ORPHAN CASE. The sweep used to check only `res.ok`, then delete the + // D1 rows — which are the only record of whose bytes a vault held. A 200 + // from anything other than the real handler would have left the storage + // billed, unattributable, and unreachable by any retry. + const acct = await seedAccount(`purge-liar-${kind.replace(/[^a-z0-9]+/gi, "-")}@example.com`, { + vaults: ["notreallygone"], + }); + await seedVaultMirrors("notreallygone"); + await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + + const liar: VaultFetch = async () => Response.json(body, { status: 200 }); + const summary = await runAccountDeleteSweep( + env, + accountDeps(undefined, liar), + plus(DELETE_UNDO_WINDOW_MS), + ); + expect(summary).toEqual({ due: 1, purged: 0, deferred: 1, failed: 0 }); + // Everything that names the vault survives, so a later real destroy can + // still find it. + expect(await countRows("vaults", "name", "notreallygone")).toBe(1); + expect(await countRows("vault_usage", "vault_name", "notreallygone")).toBe(1); + expect((await getUserById(env.DB, acct.userId))?.deletedAt).toBe(T0.toISOString()); + }, + ); + + test("an unparseable 200 body also defers", async () => { + const acct = await seedAccount("purge-unparseable@example.com", { vaults: ["garbled"] }); + await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + const liar: VaultFetch = async () => new Response("not json", { status: 200 }); + const summary = await runAccountDeleteSweep(env, accountDeps(undefined, liar), plus(DELETE_UNDO_WINDOW_MS)); + expect(summary).toEqual({ due: 1, purged: 0, deferred: 1, failed: 0 }); + expect(await countRows("vaults", "name", "garbled")).toBe(1); + }); + + test("each failed pass increments the attempt counter, and past the threshold it pages the operator", async () => { + const acct = await seedAccount("purge-escalate@example.com", { vaults: ["forever-wedged"] }); + await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + const { vaultFetch } = recordingVaultFetch({ failFor: new Set(["forever-wedged"]) }); + const { sender, ops } = recordingSender(); + const alertEnv = { ...env, OPERATOR_ALERT_EMAIL: "ops@example.com", ENVIRONMENT: "test-env" }; + + // Wind the counter to one below the threshold, then take the two passes + // that straddle it — the second is the one that must escalate. + await env.DB + .prepare("UPDATE users SET delete_purge_attempts = ? WHERE id = ?") + .bind(ACCOUNT_PURGE_ALERT_AFTER_ATTEMPTS - 2, acct.userId) + .run(); + + const quiet = await runAccountDeleteSweep( + alertEnv, + accountDeps(undefined, vaultFetch), + plus(DELETE_UNDO_WINDOW_MS), + { sender }, + ); + expect(quiet.deferred).toBe(1); + expect((await getUserById(env.DB, acct.userId))?.deletePurgeAttempts).toBe( + ACCOUNT_PURGE_ALERT_AFTER_ATTEMPTS - 1, + ); + // Below the threshold the operator is NOT woken — otherwise a transient + // fault would page on its first pass and train them to ignore it. + expect(ops).toHaveLength(0); + + const loud = await runAccountDeleteSweep( + alertEnv, + accountDeps(undefined, vaultFetch), + plus(DELETE_UNDO_WINDOW_MS + 3_600_000), + { sender }, + ); + expect(loud.deferred).toBe(1); + expect((await getUserById(env.DB, acct.userId))?.deletePurgeAttempts).toBe( + ACCOUNT_PURGE_ALERT_AFTER_ATTEMPTS, + ); + expect(ops).toHaveLength(1); + expect(ops[0]!.to).toBe("ops@example.com"); + expect(ops[0]!.subject).toContain("account deletion stuck"); + // The alert must carry the account id — an operator cannot act on "an + // account somewhere is stuck". + expect(ops[0]!.text).toContain(acct.userId); + // …and must say the data is still there, which is the whole reason it is + // an email and not a log line. + expect(ops[0]!.text).toContain("STILL PRESENT"); + + // Dedupe: another pass inside the hour does not re-page. + await runAccountDeleteSweep( + alertEnv, + accountDeps(undefined, vaultFetch), + plus(DELETE_UNDO_WINDOW_MS + 3_600_000 + 60_000), + { sender }, + ); + expect(ops).toHaveLength(1); + }); + + test("a converging account never accrues attempts", async () => { + // The negative control for the counter: if it incremented on success, the + // escalation above would eventually page for healthy deletions. + const acct = await seedAccount("purge-noattempts@example.com", { vaults: ["smooth"] }); + await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + const summary = await runAccountDeleteSweep(env, accountDeps(), plus(DELETE_UNDO_WINDOW_MS)); + expect(summary).toEqual({ due: 1, purged: 1, deferred: 0, failed: 0 }); + expect(await getUserById(env.DB, acct.userId)).toBeNull(); + }); + + test("a live (never-deleted) account is invisible to the sweep — the negative control", async () => { + const live = await seedAccount("purge-control@example.com", { vaults: ["safe"] }); + const { vaultFetch, calls } = recordingVaultFetch(); + const summary = await runAccountDeleteSweep( + env, + accountDeps(undefined, vaultFetch), + plus(DELETE_UNDO_WINDOW_MS * 10), + ); + expect(summary).toEqual({ due: 0, purged: 0, deferred: 0, failed: 0 }); + expect(calls).toEqual([]); + expect((await getUserById(env.DB, live.userId)) !== null).toBe(true); + expect(await countRows("vaults", "name", "safe")).toBe(1); + }); + + test("a throwing drip does not starve the sweep on the same tick", async () => { + // The drip runs FIRST on the hourly tick. Unguarded, a permanently + // throwing drip meant no deleted account was ever purged — the promise in + // the deletion email quietly never kept, with nothing failing loudly. + const acct = await seedAccount("purge-drip-throws@example.com"); + await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + + // A sender whose drip send throws (rather than returning !ok) is the + // cheapest way to make runDrip itself reject through the seam it owns. + const exploding: EmailSender = { + kind: "devlog", + async sendMagicLink(): Promise { + return { ok: true }; + }, + async sendOps(): Promise { + return { ok: true }; + }, + async sendDrip(): Promise { + throw new Error("drip is broken"); + }, + }; + // A LIVE, welcome-window arrival, so runDrip actually reaches the sender + // and throws. It cannot be the deleted account: A-1 excludes a tombstoned + // row from every drip eligibility query, so a deleted user is by + // construction never drip-eligible. (This is precisely how the first + // version of this test came out vacuous — the mutation that removes the + // guard passed it, because the sender was never called at all.) + const tick = plus(DELETE_UNDO_WINDOW_MS); + const live = await seedUser("purge-drip-live@example.com"); + await env.DB + .prepare("UPDATE users SET created_at = ? WHERE id = ?") + .bind(new Date(tick.getTime() - 60_000).toISOString(), live.id) + .run(); + // Control: the drip really is about to fire for this user. + expect((await eligibleFor(env.DB, "welcome", tick, 5)).map((u) => u.id)).toContain(live.id); + + await handleScheduled(DRIP_CRON, env, exploding, { now: () => tick }); + // The tick survived the drip's exception and the sweep still ran. + expect(await getUserById(env.DB, acct.userId)).toBeNull(); + }); + + test("the sweep is actually ON the hourly cron — not merely exported", async () => { + // Without this, every assertion above would hold against a function no + // deployment ever calls. Drives the real scheduled entrypoint. + const acct = await seedAccount("purge-cron@example.com"); + await handleAccountDelete(env, deleteReq(acct.token, acct.email), accountDeps(clock(T0))); + + await handleScheduled(DRIP_CRON, env, recordingSender().sender, { + now: () => plus(DELETE_UNDO_WINDOW_MS), + }); + expect(await getUserById(env.DB, acct.userId)).toBeNull(); + }); +}); diff --git a/workers/identity/test/conformance.test.ts b/workers/identity/test/conformance.test.ts index 8be4b01..7db89e7 100644 --- a/workers/identity/test/conformance.test.ts +++ b/workers/identity/test/conformance.test.ts @@ -129,6 +129,18 @@ describe("discovery endpoints", () => { // ceremony that also carries password + next. expect(md.auth).toEqual({ methods: ["magic_link"], signin_path: "/login" }); expect((md.capabilities as Record).vault_rename).toBe(false); + // cloud#226: the delete door is real (DELETE /account/vaults/ — vault + // destroy + identity D1 sweep), so the descriptor must advertise it. This + // flag is the ONLY thing a client reads to decide whether to offer delete; + // an unadvertised working route ships dark. + expect((md.capabilities as Record).vault_delete).toBe(true); + // Its whole-account sibling (A-3) has NO flag yet, and that absence is + // pinned rather than left ambiguous: `AccountCapabilities` is the shared, + // CI-pinned door contract from parachute-hub, so adding a key is an + // upstream PR. This assertion is what will fail — loudly, and in the right + // place — the day someone widens the contract and forgets to light the + // cloud door up. + expect((md.capabilities as Record).account_delete).toBeUndefined(); expect(Array.isArray(md.plans) && (md.plans as unknown[]).length).toBe(4); // PR-2: the {name}-placeholder vault-URL template, derived from vaultInstanceUrl. expect(md.vault_url_template).toContain("{name}");