From 969995c74e9124946b5ba423643df4eb4a46bb60 Mon Sep 17 00:00:00 2001 From: soonwoo Date: Thu, 3 Sep 2026 09:41:05 +0900 Subject: [PATCH] fix : drain builder before worker deploy --- .github/workflows/deploy-bloom-worker.yml | 43 ++++++++++++++++++++--- scripts/production-runtime.policy-test.js | 27 ++++++++++++++ 2 files changed, 66 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-bloom-worker.yml b/.github/workflows/deploy-bloom-worker.yml index 04003068..752076fb 100644 --- a/.github/workflows/deploy-bloom-worker.yml +++ b/.github/workflows/deploy-bloom-worker.yml @@ -63,7 +63,7 @@ jobs: username: ubuntu password: ${{ secrets.SSH_PASSWORD }} source: "bloom-runtime/target/release/bloom-runtime-bridge" - target: "/home/ubuntu/bloombouquet/bloom-runtime/target/release/" + target: "/home/ubuntu/bloombouquet/.deploy/bloom-worker/${{ github.run_id }}/" strip_components: 3 - name: Provision and start Bloom workers @@ -91,6 +91,39 @@ jobs: fi echo "Bloom worker deployment lock acquired" + STAGING_DIR=/home/ubuntu/bloombouquet/.deploy/bloom-worker/${{ github.run_id }} + STAGED_BRIDGE="$STAGING_DIR/bloom-runtime-bridge" + LIVE_BRIDGE=/home/ubuntu/bloombouquet/bloom-runtime/target/release/bloom-runtime-bridge + NEXT_BRIDGE="$LIVE_BRIDGE.next-${{ github.run_id }}" + BLOOM_BUILDER_DRAIN_FILE=/tmp/bloom-builder-worker.drain + BLOOM_BUILDER_BUSY_FILE=/tmp/bloom-builder-worker.busy + cleanup_bloom_worker_deploy() { + rm -f "$BLOOM_BUILDER_DRAIN_FILE" "$NEXT_BRIDGE" + rm -rf "$STAGING_DIR" + } + trap cleanup_bloom_worker_deploy EXIT + trap 'exit 129' HUP + trap 'exit 130' INT + trap 'exit 143' TERM + test -f "$STAGED_BRIDGE" || { echo "Staged Bloom runtime bridge is missing"; exit 1; } + touch "$BLOOM_BUILDER_DRAIN_FILE" + echo "Bloom Builder drain requested; waiting for active cycle to finish" + DRAIN_DEADLINE=$(( $(date +%s) + 7200 )) + while [ -e "$BLOOM_BUILDER_BUSY_FILE" ]; do + BUSY_PID="$(head -n 1 "$BLOOM_BUILDER_BUSY_FILE" 2>/dev/null || true)" + if [ -n "$BUSY_PID" ] && ! kill -0 "$BUSY_PID" 2>/dev/null; then + echo "Removing stale Bloom Builder busy marker for PID $BUSY_PID" + rm -f "$BLOOM_BUILDER_BUSY_FILE" + break + fi + if [ "$(date +%s)" -ge "$DRAIN_DEADLINE" ]; then + echo "Timed out waiting for active Bloom Builder cycle to drain" + exit 1 + fi + sleep 5 + done + echo "Bloom Builder drain complete" + BLOOM_SWAP_FILE=/swapfile if ! swapon --show=NAME --noheadings --raw | grep -Fxq "$BLOOM_SWAP_FILE"; then if [ ! -f "$BLOOM_SWAP_FILE" ]; then @@ -140,10 +173,12 @@ jobs: pnpm install --frozen-lockfile pnpm run build:bloom-worker test -f /home/ubuntu/bloombouquet/.tmp/bloom-worker/bloomLocalAgentRuntime.js || { echo "Bloom Local Agent runner is missing"; exit 1; } - chmod 755 bloom-runtime/target/release/bloom-runtime-bridge - test -x bloom-runtime/target/release/bloom-runtime-bridge || { echo "Bloom runtime bridge is missing"; exit 1; } - BRIDGE_SMOKE_OUTPUT="$(printf '%s' '{}' | bloom-runtime/target/release/bloom-runtime-bridge 2>/dev/null || true)" + install -d "$(dirname "$LIVE_BRIDGE")" + install -m 755 "$STAGED_BRIDGE" "$NEXT_BRIDGE" + test -x "$NEXT_BRIDGE" || { echo "Bloom runtime bridge candidate is missing"; exit 1; } + BRIDGE_SMOKE_OUTPUT="$(printf '%s' '{}' | "$NEXT_BRIDGE" 2>/dev/null || true)" printf '%s' "$BRIDGE_SMOKE_OUTPUT" | grep -Fq '"ok":false' || { echo "Bloom Runtime bridge execution smoke failed"; exit 1; } + mv -f "$NEXT_BRIDGE" "$LIVE_BRIDGE" echo "Bloom Runtime bridge execution smoke OK" ENV_FILE=/home/ubuntu/bloombouquet/.env diff --git a/scripts/production-runtime.policy-test.js b/scripts/production-runtime.policy-test.js index 00b3ca2d..0a0fdc61 100644 --- a/scripts/production-runtime.policy-test.js +++ b/scripts/production-runtime.policy-test.js @@ -163,6 +163,33 @@ test('production Bloom worker serializes remote provision runs before touching t assert.notEqual(fetchMain, -1, 'remote provision must still refresh main'); assert.ok(lockFd < fetchMain && lockWait < fetchMain, 'deployment lock must be acquired before touching the shared checkout'); }); +test('production Bloom worker drains active Builder work before mutating live runtime files', () => { + const workflow = readBloomWorkerDeployWorkflow(); + const stagedTransfer = workflow.indexOf('/home/ubuntu/bloombouquet/.deploy/bloom-worker/${{ github.run_id }}/'); + const drainTouch = workflow.indexOf('touch "$BLOOM_BUILDER_DRAIN_FILE"'); + const busyWait = workflow.indexOf('while [ -e "$BLOOM_BUILDER_BUSY_FILE" ]'); + const fetchMain = workflow.indexOf('git fetch origin main'); + const installNext = workflow.indexOf('install -m 755 "$STAGED_BRIDGE" "$NEXT_BRIDGE"'); + const smokeNext = workflow.indexOf('BRIDGE_SMOKE_OUTPUT="$(printf'); + const smokeNextBinary = workflow.indexOf('| "$NEXT_BRIDGE" 2>/dev/null'); + const promoteBridge = workflow.indexOf('mv -f "$NEXT_BRIDGE" "$LIVE_BRIDGE"'); + + assert.notEqual(stagedTransfer, -1, 'runtime bridge must transfer into a staging directory'); + assert.notEqual(drainTouch, -1, 'deployment must request Builder drain before live mutation'); + assert.notEqual(busyWait, -1, 'deployment must wait until the active Builder cycle is no longer busy'); + assert.notEqual(installNext, -1, 'staged runtime bridge must be copied to a next path after drain'); + assert.notEqual(smokeNext, -1, 'next runtime bridge smoke command must exist'); + assert.notEqual(smokeNextBinary, -1, 'next runtime bridge must be the binary under smoke'); + assert.notEqual(promoteBridge, -1, 'verified next runtime bridge must be atomically promoted'); + assert.ok(drainTouch < busyWait && busyWait < fetchMain, 'drain must complete before touching the shared checkout'); + assert.ok(busyWait < installNext && installNext < smokeNext && smokeNext < promoteBridge, 'bridge promotion must happen only after idle and smoke verification'); + assert.match(workflow, /trap cleanup_bloom_worker_deploy EXIT/); + assert.match(workflow, /trap 'exit 129' HUP/); + assert.match(workflow, /trap 'exit 130' INT/); + assert.match(workflow, /trap 'exit 143' TERM/); + assert.match(workflow, /rm -f "\$BLOOM_BUILDER_DRAIN_FILE"/); +}); + test('production Bloom worker provisions emergency swap before starting memory-heavy local inference', () => { const workflow = readBloomWorkerDeployWorkflow();