Fence fleet reconciliation during app removal (#4)

Co-authored-by: gitcommit90 <gitcommit90@users.noreply.github.com>

Author: gitcommit90

CHANGELOG.md

@@ -39,6 +39,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   backgrounding or transport loss while retaining the same server session,
   shell state, working directory, and scrollback. Disconnect text is no longer
   written into the terminal.
+- App-removal preparation now quiesces and fences automatic fleet care before
+  deleting owned Apple channel machines, so an in-flight reconciliation pass
+  cannot recreate a machine from stale pre-removal state.
 
 ## [0.0.3] - 2026-07-24
 

src/server/channel-computers.ts

@@ -72,6 +72,7 @@ const CONTAINER_CANDIDATES = [process.env.HELM_CONTAINER_CLI, "/usr/local/bin/co
 const COMMAND_TIMEOUT_MS = Math.max(5_000, Number(process.env.HELM_MACHINE_COMMAND_TIMEOUT_MS || 120_000));
 const IDLE_AFTER_MS = Math.max(60_000, Number(process.env.HELM_MACHINE_IDLE_MS || 15 * 60_000));
 const RECONCILE_EVERY_MS = Math.max(15_000, Number(process.env.HELM_FLEET_INTERVAL_MS || 60_000));
+const INITIAL_RECONCILE_MS = Math.max(25, Number(process.env.HELM_FLEET_INITIAL_MS || 2_000));
 const UPDATE_EVERY_MS = Math.max(24 * 60 * 60_000, Number(process.env.HELM_MACHINE_UPDATE_MS || 7 * 24 * 60 * 60_000));
 const UPDATE_RETRY_MS = Math.max(60 * 60_000, Number(process.env.HELM_MACHINE_UPDATE_RETRY_MS || 6 * 60 * 60_000));
 const MAX_WORKSPACE_SYNC_BYTES = Math.max(64 * 1024 ** 2, Number(process.env.HELM_WORKSPACE_SYNC_MAX_BYTES || 2 * 1024 ** 3));
@@ -85,7 +86,10 @@ const terminalSessions = new Map<string, MachineTerminal>();
 const channelLocks = new Map<number, Promise<unknown>>();
 const syncTimers = new Map<number, NodeJS.Timeout>();
 let reconcileTimer: NodeJS.Timeout | null = null;
+let reconcileStartupTimer: NodeJS.Timeout | null = null;
 let reconcileRunning = false;
+let reconcileEnabled = false;
+let reconcilePass: Promise<void> | null = null;
 
 const installationId = (): string => {
   let id = String(q1("SELECT installation_id FROM workspace WHERE id=1")?.installation_id || "");
@@ -757,6 +761,10 @@ export async function appRemovalStatus(): Promise<{ backend: ChannelComputerBack
 export async function prepareAppRemoval(): Promise<{ backend: ChannelComputerBackend; deleted: number; remaining: number }> {
   const backend = configuredChannelBackend();
   if (backend !== "apple") return { backend, deleted: 0, remaining: 0 };
+  // Uninstall is a fleet-wide terminal state. Quiesce and fence the reconciler
+  // before enumerating machines so an already-running pass cannot recreate a
+  // machine from its stale pre-removal snapshot after deletion completes.
+  await shutdownChannelComputers();
   const install = installationId();
   const prefix = `1helm-${install}-channel-`;
   const machineIds = await ownedInstallationMachineIds();
@@ -1028,19 +1036,32 @@ export async function reconcileChannelComputers(channelIds?: Iterable<number>):
 }
 
 export function startChannelComputerReconciler(): void {
-  if (reconcileTimer) return;
+  if (reconcileEnabled) return;
+  reconcileEnabled = true;
   const tick = (): void => {
-    if (reconcileRunning) return;
+    if (!reconcileEnabled || reconcileRunning) return;
     reconcileRunning = true;
-    void reconcileChannelComputers().catch((error) => console.error("channel computer reconcile failed:", (error as Error).message)).finally(() => { reconcileRunning = false; });
+    const pass: Promise<void> = reconcileChannelComputers()
+      .then(() => undefined)
+      .catch((error) => console.error("channel computer reconcile failed:", (error as Error).message))
+      .finally(() => {
+        reconcileRunning = false;
+        if (reconcilePass === pass) reconcilePass = null;
+      });
+    reconcilePass = pass;
+    void pass;
   };
-  setTimeout(tick, 2_000).unref();
+  reconcileStartupTimer = setTimeout(() => { reconcileStartupTimer = null; tick(); }, INITIAL_RECONCILE_MS);
+  reconcileStartupTimer.unref();
   reconcileTimer = setInterval(tick, RECONCILE_EVERY_MS);
   reconcileTimer.unref();
 }
 
 export async function shutdownChannelComputers(): Promise<void> {
+  reconcileEnabled = false;
+  if (reconcileStartupTimer) { clearTimeout(reconcileStartupTimer); reconcileStartupTimer = null; }
   if (reconcileTimer) { clearInterval(reconcileTimer); reconcileTimer = null; }
+  await reconcilePass?.catch(() => undefined);
   for (const session of [...terminalSessions.values()]) closeChannelTerminal(session.id);
   for (const timer of syncTimers.values()) clearTimeout(timer);
   syncTimers.clear();

test/channel-computers.mjs

@@ -19,6 +19,7 @@ process.env.HELM_CHANNEL_COMPUTER_BACKEND = "apple";
 process.env.HELM_CONTAINER_CLI = fakeCli;
 process.env.FAKE_CONTAINER_STATE = fakeState;
 process.env.HELM_FLEET_INTERVAL_MS = "600000";
+process.env.HELM_FLEET_INITIAL_MS = "25";
 process.env.HELM_MACHINE_IDLE_MS = "60000";
 
 const db = await import("../src/server/db.ts");
@@ -143,9 +144,16 @@ test("Apple channel-computer contract preserves isolation, files, wakes, archive
   const backend = await readFile(join(root, "src", "server", "channel-computers.ts"), "utf8");
   assert.match(backend, /machine", "run", "-it"[\s\S]*guestWords\("\/bin\/bash", "-l"\)/, "interactive Apple terminals request an explicit guest login shell");
 
+  // Reproduce the real Apple race: uninstall begins while an automatic fleet
+  // pass is already inspecting a machine. Removal must wait for that pass,
+  // fence future ticks, and leave nothing for a stale snapshot to recreate.
+  process.env.FAKE_CONTAINER_INSPECT_DELAY_MS = "200";
+  computers.startChannelComputerReconciler();
+  await new Promise((resolveWait) => setTimeout(resolveWait, 50));
   const removal = await computers.prepareAppRemoval();
   assert.equal(removal.deleted, 1, "uninstall preparation deletes every remaining VM owned by this exact 1Helm installation");
   assert.equal(removal.remaining, 0);
+  await new Promise((resolveWait) => setTimeout(resolveWait, 100));
   assert.equal(existsSync(join(fakeState, "machines", betaComputer.machine_id)), false, "no owned channel VM survives uninstall preparation");
   computers.reactivateComputersAfterPreparedRemoval();
   assert.equal(db.q1("SELECT desired_state FROM channel_computers WHERE channel_id=?", beta.channelId).desired_state, "auto", "a later reinstall can rebuild the removed VM from its preserved host mirror");

test/fake-container.mjs

@@ -63,6 +63,8 @@ if (action === "create") {
 const config = readConfig(name);
 if (!config) fail(`machine ${name} not found`);
 if (action === "inspect") {
+  const delay = Math.max(0, Number(process.env.FAKE_CONTAINER_INSPECT_DELAY_MS || 0));
+  if (delay) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, delay);
   process.stdout.write(JSON.stringify(config));
   process.exit(0);
 }