fix: drain builder before mutating worker runtime

.github/workflows/deploy-bloom-worker.yml

@@ -63,7 +63,7 @@ jobs:
           username: ubuntu
           password: ${{ secrets.SSH_PASSWORD }}
           source: "bloom-runtime/target/release/bloom-runtime-bridge"
-          target: "/home/ubuntu/bloombouquet/bloom-runtime/target/release/"
+          target: "/home/ubuntu/bloombouquet/.deploy/bloom-worker/${{ github.run_id }}/"
           strip_components: 3
 
       - name: Provision and start Bloom workers
@@ -91,6 +91,39 @@ jobs:
             fi
             echo "Bloom worker deployment lock acquired"
 
+            STAGING_DIR=/home/ubuntu/bloombouquet/.deploy/bloom-worker/${{ github.run_id }}
+            STAGED_BRIDGE="$STAGING_DIR/bloom-runtime-bridge"
+            LIVE_BRIDGE=/home/ubuntu/bloombouquet/bloom-runtime/target/release/bloom-runtime-bridge
+            NEXT_BRIDGE="$LIVE_BRIDGE.next-${{ github.run_id }}"
+            BLOOM_BUILDER_DRAIN_FILE=/tmp/bloom-builder-worker.drain
+            BLOOM_BUILDER_BUSY_FILE=/tmp/bloom-builder-worker.busy
+            cleanup_bloom_worker_deploy() {
+              rm -f "$BLOOM_BUILDER_DRAIN_FILE" "$NEXT_BRIDGE"
+              rm -rf "$STAGING_DIR"
+            }
+            trap cleanup_bloom_worker_deploy EXIT
+            trap 'exit 129' HUP
+            trap 'exit 130' INT
+            trap 'exit 143' TERM
+            test -f "$STAGED_BRIDGE" || { echo "Staged Bloom runtime bridge is missing"; exit 1; }
+            touch "$BLOOM_BUILDER_DRAIN_FILE"
+            echo "Bloom Builder drain requested; waiting for active cycle to finish"
+            DRAIN_DEADLINE=$(( $(date +%s) + 7200 ))
+            while [ -e "$BLOOM_BUILDER_BUSY_FILE" ]; do
+              BUSY_PID="$(head -n 1 "$BLOOM_BUILDER_BUSY_FILE" 2>/dev/null || true)"
+              if [ -n "$BUSY_PID" ] && ! kill -0 "$BUSY_PID" 2>/dev/null; then
+                echo "Removing stale Bloom Builder busy marker for PID $BUSY_PID"
+                rm -f "$BLOOM_BUILDER_BUSY_FILE"
+                break
+              fi
+              if [ "$(date +%s)" -ge "$DRAIN_DEADLINE" ]; then
+                echo "Timed out waiting for active Bloom Builder cycle to drain"
+                exit 1
+              fi
+              sleep 5
+            done
+            echo "Bloom Builder drain complete"
+
             BLOOM_SWAP_FILE=/swapfile
             if ! swapon --show=NAME --noheadings --raw | grep -Fxq "$BLOOM_SWAP_FILE"; then
               if [ ! -f "$BLOOM_SWAP_FILE" ]; then
@@ -140,10 +173,12 @@ jobs:
             pnpm install --frozen-lockfile
             pnpm run build:bloom-worker
             test -f /home/ubuntu/bloombouquet/.tmp/bloom-worker/bloomLocalAgentRuntime.js || { echo "Bloom Local Agent runner is missing"; exit 1; }
-            chmod 755 bloom-runtime/target/release/bloom-runtime-bridge
-            test -x bloom-runtime/target/release/bloom-runtime-bridge || { echo "Bloom runtime bridge is missing"; exit 1; }
-            BRIDGE_SMOKE_OUTPUT="$(printf '%s' '{}' | bloom-runtime/target/release/bloom-runtime-bridge 2>/dev/null || true)"
+            install -d "$(dirname "$LIVE_BRIDGE")"
+            install -m 755 "$STAGED_BRIDGE" "$NEXT_BRIDGE"
+            test -x "$NEXT_BRIDGE" || { echo "Bloom runtime bridge candidate is missing"; exit 1; }
+            BRIDGE_SMOKE_OUTPUT="$(printf '%s' '{}' | "$NEXT_BRIDGE" 2>/dev/null || true)"
             printf '%s' "$BRIDGE_SMOKE_OUTPUT" | grep -Fq '"ok":false' || { echo "Bloom Runtime bridge execution smoke failed"; exit 1; }
+            mv -f "$NEXT_BRIDGE" "$LIVE_BRIDGE"
             echo "Bloom Runtime bridge execution smoke OK"
 
             ENV_FILE=/home/ubuntu/bloombouquet/.env

scripts/production-runtime.policy-test.js

@@ -163,6 +163,33 @@ test('production Bloom worker serializes remote provision runs before touching t
   assert.notEqual(fetchMain, -1, 'remote provision must still refresh main');
   assert.ok(lockFd < fetchMain && lockWait < fetchMain, 'deployment lock must be acquired before touching the shared checkout');
 });
+test('production Bloom worker drains active Builder work before mutating live runtime files', () => {
+  const workflow = readBloomWorkerDeployWorkflow();
+  const stagedTransfer = workflow.indexOf('/home/ubuntu/bloombouquet/.deploy/bloom-worker/${{ github.run_id }}/');
+  const drainTouch = workflow.indexOf('touch "$BLOOM_BUILDER_DRAIN_FILE"');
+  const busyWait = workflow.indexOf('while [ -e "$BLOOM_BUILDER_BUSY_FILE" ]');
+  const fetchMain = workflow.indexOf('git fetch origin main');
+  const installNext = workflow.indexOf('install -m 755 "$STAGED_BRIDGE" "$NEXT_BRIDGE"');
+  const smokeNext = workflow.indexOf('BRIDGE_SMOKE_OUTPUT="$(printf');
+  const smokeNextBinary = workflow.indexOf('| "$NEXT_BRIDGE" 2>/dev/null');
+  const promoteBridge = workflow.indexOf('mv -f "$NEXT_BRIDGE" "$LIVE_BRIDGE"');
+
+  assert.notEqual(stagedTransfer, -1, 'runtime bridge must transfer into a staging directory');
+  assert.notEqual(drainTouch, -1, 'deployment must request Builder drain before live mutation');
+  assert.notEqual(busyWait, -1, 'deployment must wait until the active Builder cycle is no longer busy');
+  assert.notEqual(installNext, -1, 'staged runtime bridge must be copied to a next path after drain');
+  assert.notEqual(smokeNext, -1, 'next runtime bridge smoke command must exist');
+  assert.notEqual(smokeNextBinary, -1, 'next runtime bridge must be the binary under smoke');
+  assert.notEqual(promoteBridge, -1, 'verified next runtime bridge must be atomically promoted');
+  assert.ok(drainTouch < busyWait && busyWait < fetchMain, 'drain must complete before touching the shared checkout');
+  assert.ok(busyWait < installNext && installNext < smokeNext && smokeNext < promoteBridge, 'bridge promotion must happen only after idle and smoke verification');
+  assert.match(workflow, /trap cleanup_bloom_worker_deploy EXIT/);
+  assert.match(workflow, /trap 'exit 129' HUP/);
+  assert.match(workflow, /trap 'exit 130' INT/);
+  assert.match(workflow, /trap 'exit 143' TERM/);
+  assert.match(workflow, /rm -f "\$BLOOM_BUILDER_DRAIN_FILE"/);
+});
+
 test('production Bloom worker provisions emergency swap before starting memory-heavy local inference', () => {
   const workflow = readBloomWorkerDeployWorkflow();