From a045686e7ee53d88f6b3362a4c7188a916c4ebd9 Mon Sep 17 00:00:00 2001 From: Regan Bell Date: Wed, 29 Jul 2026 17:13:15 -0700 Subject: [PATCH] Warn after fly secrets push when staged secrets are not live on running machines In a real deployment an operator ran qm secrets push, then restarted the Fly machines expecting the new secrets to apply. Fly staged secrets are only applied by a deploy or machine update, not a restart, so the auth service ran without SMTP credentials until a full qm up. The existing staging output did not make this failure mode obvious. flySecretsPush now checks each app that received staged secrets and, when any of them has running machines, ends with an explicit warning that the staged secrets are not live and that qm up (not a machine restart) applies them. Fresh installs with no machines see no warning. Co-Authored-By: Claude Fable 5 --- cli/src/backends/fly.ts | 16 ++++++ cli/test/fly-sandbox.test.ts | 108 +++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/cli/src/backends/fly.ts b/cli/src/backends/fly.ts index 42b062971..97ee5e4c3 100644 --- a/cli/src/backends/fly.ts +++ b/cli/src/backends/fly.ts @@ -408,6 +408,15 @@ function flyOrgApps(flyOrg: string): Set { ); } +function appHasMachines(app: string): boolean { + try { + const parsed = JSON.parse(fly(["status", "-a", app, "--json"])) as { Machines?: unknown[] }; + return (parsed.Machines?.length ?? 0) > 0; + } catch { + return false; + } +} + function ensureApp(app: string, flyOrg: string, orgId: string, appPrefix: string): void { const out = fly(["apps", "create", app, "--org", flyOrg], { allow: /already been taken/i }); const marker = flyOwnershipMarker(flyOrg, orgId, appPrefix); @@ -1620,6 +1629,7 @@ export async function flySecretsPush(config: QmConfig, configDir: string, envFil for (const plugin of pluginNames) ensureApp(`${prefix}-${plugin}`, ctx.flyOrg, ctx.orgId, ctx.appPrefix); unsetDisabledSecurityScreenToken(config, prefix); unsetDisabledFlyPublisherToken(config, prefix); + const stagedApps = new Set(); for (const secret of operatorSecrets) { const supplied = deploymentSecretValue(secret.name, values.get(secret.name)); if (!secret.required && !supplied) { @@ -1635,6 +1645,7 @@ export async function flySecretsPush(config: QmConfig, configDir: string, envFil destinations.set(`${prefix}-${workload}`, names); } for (const [app, names] of destinations) { + stagedApps.add(app); for (const name of names) { stageSecret(app, name, value); } @@ -1642,4 +1653,9 @@ export async function flySecretsPush(config: QmConfig, configDir: string, envFil step(`${secret.name}: staged on ${[...destinations].map(([app]) => app).join(", ")}`); } ok("operator secrets staged on Fly"); + const running = [...stagedApps].filter(appHasMachines); + if (running.length) { + warn(`staged secrets are NOT live yet on ${running.join(", ")}: running machines keep their old values`); + warn("run `qm up` to apply them — a plain machine restart does not"); + } } diff --git a/cli/test/fly-sandbox.test.ts b/cli/test/fly-sandbox.test.ts index 1150c1392..4d1d7f28c 100644 --- a/cli/test/fly-sandbox.test.ts +++ b/cli/test/fly-sandbox.test.ts @@ -306,6 +306,114 @@ test("fly secrets push stages a dual-role secret under BOTH names on the core ap } }); +test("fly secrets push warns that staged secrets are not live when machines are running", async () => { + const dir = mkdtempSync(join(tmpdir(), "qm-fly-push-staged-warn-")); + const config: QmConfig = { + contract: 1, + orgId: "acme", + publicUrl: "https://acme.example.com", + target: "fly", + region: "sjc", + flyOrg: "personal", + services: ["core"], + plugins: [], + skills: [], + env: { core: { HARNESS: "mock" } }, + imageOverrides: {}, + sandbox: { app: "acme-sb" }, + }; + writeFileSync( + join(dir, ".env"), + [ + "CAPABILITY_SECRET=capability-secret-that-is-long-enough", + `CONNECTOR_SECRET_KEY=${"connector".repeat(4)}`, + `CORE_SIGNING_SECRET=${"core-signing".repeat(3)}`, + "PORTAL_IDENTITY_SECRET=portal-identity-secret-that-is-long-enough", + `SKILL_SIGNING_SECRET=${"skill-signing".repeat(3)}`, + "FLY_SANDBOX_API_TOKEN=fly", + ].join("\n"), + ); + const fake = fakeFly( + dir, + ` +if (a.startsWith("status -a acme-core")) console.log(JSON.stringify({ Machines: [{ id: "m1", state: "started" }] })); +else if (a.startsWith("secrets set ")) fs.readFileSync(0, "utf8"); +`, + ); + const log = console.log; + const warnLog = console.warn; + const warnings: string[] = []; + console.log = (): void => {}; + console.warn = (msg: string): void => { + warnings.push(msg); + }; + try { + await flySecretsPush(config, dir); + assert.ok( + warnings.some((line) => line.includes("staged secrets are NOT live yet on acme-core")), + warnings.join("\n"), + ); + assert.ok(warnings.some((line) => line.includes("run `qm up`"))); + } finally { + console.log = log; + console.warn = warnLog; + fake.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("fly secrets push stays quiet about staging when no machines are running", async () => { + const dir = mkdtempSync(join(tmpdir(), "qm-fly-push-staged-quiet-")); + const config: QmConfig = { + contract: 1, + orgId: "acme", + publicUrl: "https://acme.example.com", + target: "fly", + region: "sjc", + flyOrg: "personal", + services: ["core"], + plugins: [], + skills: [], + env: { core: { HARNESS: "mock" } }, + imageOverrides: {}, + sandbox: { app: "acme-sb" }, + }; + writeFileSync( + join(dir, ".env"), + [ + "CAPABILITY_SECRET=capability-secret-that-is-long-enough", + `CONNECTOR_SECRET_KEY=${"connector".repeat(4)}`, + `CORE_SIGNING_SECRET=${"core-signing".repeat(3)}`, + "PORTAL_IDENTITY_SECRET=portal-identity-secret-that-is-long-enough", + `SKILL_SIGNING_SECRET=${"skill-signing".repeat(3)}`, + "FLY_SANDBOX_API_TOKEN=fly", + ].join("\n"), + ); + const fake = fakeFly( + dir, + ` +if (a.startsWith("status -a")) console.log(JSON.stringify({ Machines: [] })); +else if (a.startsWith("secrets set ")) fs.readFileSync(0, "utf8"); +`, + ); + const log = console.log; + const warnLog = console.warn; + const warnings: string[] = []; + console.log = (): void => {}; + console.warn = (msg: string): void => { + warnings.push(msg); + }; + try { + await flySecretsPush(config, dir); + assert.ok(!warnings.some((line) => line.includes("staged secrets are NOT live")), warnings.join("\n")); + } finally { + console.log = log; + console.warn = warnLog; + fake.restore(); + rmSync(dir, { recursive: true, force: true }); + } +}); + test("fly secrets push removes the disabled Fly app publisher token", async () => { const dir = mkdtempSync(join(tmpdir(), "qm-fly-push-publisher-off-")); const config: QmConfig = {