diff --git a/plugins/admin/src/index.ts b/plugins/admin/src/index.ts index d0415cd81..fb0386175 100644 --- a/plugins/admin/src/index.ts +++ b/plugins/admin/src/index.ts @@ -7,6 +7,7 @@ import { signedRequestHeaders, withSourceAuthNonce } from "../../chassis/src/cor import { json, readBody, cookie } from "../../chassis/src/http.ts"; import { createBrandingCache, injectBranding, type OrgBranding } from "../../chassis/src/branding.ts"; import { verifyPortalIdentity, PORTAL_IDENTITY_HEADER } from "../../chassis/src/portal-identity.ts"; +import { errMessage } from "../../chassis/src/errors.ts"; import { CORE_API_URL as CORE, CORE_ORG_ID as ORG, @@ -20,6 +21,9 @@ import { dirname, join } from "node:path"; const PORT = portFromEnv(8090); const ADMIN_BASE_PATH = (process.env.ADMIN_BASE_PATH ?? "").replace(/\/$/, ""); +const CORE_WHOAMI_ATTEMPTS = 2; +const CORE_WHOAMI_TIMEOUT_MS = 2_500; +const CORE_WHOAMI_RETRY_DELAY_MS = 250; function signedHeaders(method: string, corePath: string, rawBody: string): Record { return signedRequestHeaders(CORE_SIGNING_SECRET, method, corePath, rawBody, { "content-type": "application/json" }); } @@ -243,19 +247,37 @@ async function uploadFileFromRequest( async function coreWhoami(principal: string): Promise<{ isAdmin: boolean; role?: string; scopeId?: string } | null> { const corePath = "/v1/admin/whoami"; - try { - const r = await fetch(`${CORE}${corePath}`, { - headers: { - ...signedHeaders("GET", corePath, ""), - ...portalIdentityHeader(), - "x-admin-actor": `${principal}@${ORG}`, - }, - }); - if (!r.ok) return null; - return (await r.json()) as { isAdmin: boolean; role?: string; scopeId?: string }; - } catch { - return null; + const started = Date.now(); + let failure = "unknown failure"; + let attempts = 0; + for (; attempts < CORE_WHOAMI_ATTEMPTS; attempts++) { + try { + const r = await fetch(`${CORE}${corePath}`, { + headers: { + ...signedHeaders("GET", corePath, ""), + ...portalIdentityHeader(), + "x-admin-actor": `${principal}@${ORG}`, + }, + signal: AbortSignal.timeout(CORE_WHOAMI_TIMEOUT_MS), + }); + if (r.ok) { + const body = (await r.json()) as { isAdmin?: unknown; role?: string; scopeId?: string }; + if (typeof body.isAdmin !== "boolean") throw new Error("core returned an invalid admin status"); + return { ...body, isAdmin: body.isAdmin }; + } + failure = `HTTP ${r.status}`; + if (r.status < 500 && r.status !== 429) break; + } catch (error) { + failure = errMessage(error); + } + if (attempts + 1 < CORE_WHOAMI_ATTEMPTS) { + await new Promise((resolve) => setTimeout(resolve, CORE_WHOAMI_RETRY_DELAY_MS)); + } } + console.warn( + `[admin] core whoami failed after ${Math.min(attempts + 1, CORE_WHOAMI_ATTEMPTS)} attempt(s) in ${Date.now() - started}ms: ${failure}`, + ); + return null; } const WRITES = new Map([ diff --git a/plugins/admin/test/whoami.test.ts b/plugins/admin/test/whoami.test.ts index 27233e453..db7762a25 100644 --- a/plugins/admin/test/whoami.test.ts +++ b/plugins/admin/test/whoami.test.ts @@ -7,8 +7,16 @@ import { mintPortalIdentity } from "../../chassis/src/portal-identity.ts"; let lastActor: string | null = null; let lastSigned = false; let lastPortalIdentity: string | null = null; +let transientWhoamiFailures = 0; +let whoamiRequests = 0; const core = createServer((req: IncomingMessage, res) => { if (req.method === "GET" && (req.url ?? "").startsWith("/v1/admin/whoami")) { + whoamiRequests++; + if (transientWhoamiFailures > 0) { + transientWhoamiFailures--; + res.writeHead(502, { "content-type": "application/json" }); + return void res.end(JSON.stringify({ error: "temporarily_unavailable" })); + } lastActor = (req.headers["x-admin-actor"] as string) ?? null; lastSigned = Boolean(req.headers["x-timestamp"] && req.headers["x-signature"]); lastPortalIdentity = (req.headers["x-portal-identity"] as string) ?? null; @@ -134,6 +142,15 @@ test("an unsigned or wrongly-signed portal identity is not accepted as an admin } }); +test("a transient core failure is retried before admin bootstrap fails", async () => { + transientWhoamiFailures = 1; + whoamiRequests = 0; + const r = await api("/api/me", "admin=U-admin"); + assert.equal(r.status, 200); + assert.equal(whoamiRequests, 2); + assert.equal(((await r.json()) as { isAdmin?: boolean }).isAdmin, true); +}); + test("core unreachable → /api/whoami returns 502 core_unreachable (outage, not a not-admin verdict)", async () => { await new Promise((r) => core.close(() => r())); const r = await api("/api/whoami", "admin=U-admin"); diff --git a/plugins/portal/src/index.ts b/plugins/portal/src/index.ts index a0abdee9d..fcb7ff941 100644 --- a/plugins/portal/src/index.ts +++ b/plugins/portal/src/index.ts @@ -201,8 +201,9 @@ export function consumeState(state: string): boolean { } const ADMIN_TTL_MS = 60_000; -const ADMIN_PROBE_TIMEOUT_MS = 1500; +const ADMIN_PROBE_TIMEOUT_MS = 6_500; const ADMIN_PROBE_ATTEMPTS = 2; +const ADMIN_PROBE_RETRY_DELAY_MS = 250; const adminCache = new LRUCache({ max: 10_000, ttl: ADMIN_TTL_MS }); async function adminProbeAttempt(sub: string): Promise { @@ -217,10 +218,18 @@ async function adminProbeAttempt(sub: string): Promise { ); } const r = await fetch(`${UPSTREAMS.admin}/api/whoami`, { headers, signal: ctrl.signal }); - if (!r.ok) return null; - const j = (await r.json()) as { isAdmin?: boolean }; - return j.isAdmin === true; - } catch { + if (!r.ok) { + console.warn(`[portal] admin probe returned HTTP ${r.status}`); + return null; + } + const j = (await r.json()) as { isAdmin?: unknown }; + if (typeof j.isAdmin !== "boolean") { + console.warn("[portal] admin probe returned an invalid admin status"); + return null; + } + return j.isAdmin; + } catch (error) { + console.warn(`[portal] admin probe failed: ${errMessage(error)}`); return null; } finally { clearTimeout(timer); @@ -232,7 +241,11 @@ async function adminProbe(sub: string): Promise<{ isAdmin: boolean; failed: bool if (hit !== undefined) return { isAdmin: hit, failed: false }; for (let attempt = 0; attempt < ADMIN_PROBE_ATTEMPTS; attempt++) { const isAdmin = await adminProbeAttempt(sub); - if (isAdmin === null) continue; + if (isAdmin === null) { + if (attempt + 1 < ADMIN_PROBE_ATTEMPTS) + await new Promise((resolve) => setTimeout(resolve, ADMIN_PROBE_RETRY_DELAY_MS)); + continue; + } adminCache.set(sub, isAdmin); return { isAdmin, failed: false }; } diff --git a/plugins/portal/test/proxy-errors.test.ts b/plugins/portal/test/proxy-errors.test.ts index 255b8d744..413e71390 100644 --- a/plugins/portal/test/proxy-errors.test.ts +++ b/plugins/portal/test/proxy-errors.test.ts @@ -5,7 +5,7 @@ import { connect } from "node:net"; import type { AddressInfo } from "node:net"; import { verifyPortalIdentity } from "../../chassis/src/portal-identity.ts"; -let whoamiMode: "ok" | "down" | "fail-once" = "ok"; +let whoamiMode: "ok" | "down" | "fail-once" | "malformed" = "ok"; let whoamiRequests = 0; const upstream = createServer((req: IncomingMessage, res) => { @@ -15,6 +15,10 @@ const upstream = createServer((req: IncomingMessage, res) => { res.writeHead(502, { "content-type": "application/json" }); return void res.end(JSON.stringify({ error: "core_unreachable" })); } + if (whoamiMode === "malformed") { + res.writeHead(200, { "content-type": "application/json" }); + return void res.end("{}"); + } const m = (req.headers.cookie ?? "").match(/admin=([^;]+)/); const sub = m ? decodeURIComponent(m[1] ?? "") : ""; res.writeHead(200, { "content-type": "application/json" }); @@ -152,6 +156,16 @@ test("an admin-probe outage is reported as unavailable and is NOT negative-cache assert.equal(ok.status, 200); }); +test("a malformed admin verdict fails readiness instead of being cached as non-admin", async () => { + whoamiMode = "malformed"; + const denied = await fetch(`${base}/admin/`, { + headers: { cookie: sessionCookie("U-admin-malformed"), accept: "text/html" }, + }); + assert.equal(denied.status, 403); + assert.match(await denied.text(), /temporarily unavailable/i); + whoamiMode = "ok"; +}); + test("consumeState: single-use, TTL-bounded, never wholesale-wiped", () => { assert.equal(consumeState("state-a"), true); assert.equal(consumeState("state-a"), false, "a consumed state cannot be replayed");