ci(compat): widen CF readiness headroom; fail the serialfilter wait

Over four compat-matrix runs the Adobe engines answered on attempt 42-57
(2023) and 48-60 (2025) of a 60-attempt ceiling, so a slightly slower runner
turns the leg red. Raise the ceiling to 120 in compat-matrix.yml and
compat-pr.yml, print "attempt N/MAX", and warn when readiness lands past 3/4
of the ceiling so drift is visible before it fails.

The post-restart wait after patching serialfilter.txt (Adobe + Oracle) fell
through silently when the engine never came back, so the suite ran against a
dead engine. Give it the READY flag from #3692, the same ceiling, and a loud
failure with container logs.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Signed-off-by: Peter Amiri <peter@alurium.com>

Author: bpamiri

.github/workflows/compat-matrix.yml

@@ -109,7 +109,14 @@ jobs:
           # Tracks the last HTTP status code so timeout diagnostics can
           # distinguish "no response" (engine didn't bind) from "5xx"
           # (engine bound but app returning errors — e.g. issue #2646).
-          MAX_WAIT=60
+          #
+          # Ceiling: measured over four matrix runs (2026-09-27), the Adobe
+          # engines answered on attempt 42-57 (2023) and 48-60 (2025) of the old
+          # 60; Lucee and BoxLang on 1-12. 120 leaves ~2x the observed worst
+          # case. Each attempt is ~5-10s, so a dead engine now fails after
+          # ~10-20 min instead of ~5-10. A ready-after-3/4 warning (below)
+          # makes future drift visible before it turns red.
+          MAX_WAIT=120
           WAIT_COUNT=0
           RESTARTS=0
           MAX_RESTARTS=3
@@ -143,7 +150,10 @@ jobs:
             LAST_HTTP_CODE=$(curl -s -o /dev/null --connect-timeout 2 --max-time 5 -w "%{http_code}" "http://localhost:${PORT}/" 2>/dev/null || true)
             LAST_HTTP_CODE=${LAST_HTTP_CODE:-000}
             if echo "$LAST_HTTP_CODE" | grep -qE "^(200|302|404)$"; then
-              echo "CF engine is ready! (HTTP $LAST_HTTP_CODE on attempt $WAIT_COUNT)"
+              echo "CF engine is ready! (HTTP $LAST_HTTP_CODE on attempt $WAIT_COUNT/$MAX_WAIT)"
+              if [ $((WAIT_COUNT * 4)) -gt $((MAX_WAIT * 3)) ]; then
+                echo "::warning::${CFENGINE} became ready at attempt $WAIT_COUNT/$MAX_WAIT (past 3/4 of the ceiling); readiness headroom is shrinking."
+              fi
               READY=true
               break
             fi
@@ -186,19 +196,35 @@ jobs:
             "echo ';oracle.sql.converter.**;oracle.sql.**;oracle.jdbc.**' >> /wheels-test-suite/.engine/${{ matrix.cfengine }}/WEB-INF/cfusion/lib/serialfilter.txt"
           docker restart wheels-${{ matrix.cfengine }}-1
 
-          # Wait for engine to come back up after restart
+          # Wait for engine to come back up after restart. This used to fall
+          # through silently when the engine never came back, so the suite ran
+          # against a dead engine; fail loudly instead. A full Adobe restart has
+          # the same cold-start profile as the readiness wait above, so it gets
+          # the same ceiling (the old 30 was never measured: it could not fail).
           PORT_VAR="PORT_${{ matrix.cfengine }}"
           PORT="${!PORT_VAR}"
-          MAX_WAIT=30
+          CONTAINER="wheels-${{ matrix.cfengine }}-1"
+          MAX_WAIT=120
           WAIT_COUNT=0
+          READY=false
           while [ "$WAIT_COUNT" -lt "$MAX_WAIT" ]; do
             WAIT_COUNT=$((WAIT_COUNT + 1))
             if curl -s -o /dev/null --connect-timeout 2 --max-time 5 -w "%{http_code}" "http://localhost:${PORT}/" | grep -q "200\|404\|302"; then
-              echo "CF engine back up after restart"
+              echo "CF engine back up after restart (attempt $WAIT_COUNT/$MAX_WAIT)"
+              READY=true
               break
             fi
-            sleep 5
+            if [ "$WAIT_COUNT" -lt "$MAX_WAIT" ]; then
+              sleep 5
+            fi
           done
+          if [ "$READY" != "true" ]; then
+            echo "::error::CF engine did not come back after the serialfilter restart (${MAX_WAIT} attempts)"
+            echo "=== Container logs (raw, last 200 lines) ==="
+            docker logs "$CONTAINER" 2>&1 | tail -200
+            echo "=== /end raw logs ==="
+            exit 1
+          fi
 
       - name: Install CFPM packages (Adobe 2023/2025)
         if: matrix.cfengine == 'adobe2023' || matrix.cfengine == 'adobe2025'

.github/workflows/compat-pr.yml

@@ -98,7 +98,11 @@ jobs:
           CONTAINER="wheels-${CFENGINE}-1"
           echo "Waiting for ${CFENGINE} on port ${PORT}..."
 
-          MAX_WAIT=60
+          # Ceiling: Adobe 2023 answered on attempt 42-57 of the old 60 across
+          # four compat-matrix runs (2026-09-27); BoxLang on 7-12. 120 leaves ~2x
+          # the observed worst case; a dead engine fails after ~10-20 min, inside
+          # this job's 45-minute timeout. See compat-matrix.yml for the details.
+          MAX_WAIT=120
           WAIT_COUNT=0
           RESTARTS=0
           MAX_RESTARTS=3
@@ -128,7 +132,10 @@ jobs:
             LAST_HTTP_CODE=$(curl -s -o /dev/null --connect-timeout 2 --max-time 5 -w "%{http_code}" "http://localhost:${PORT}/" 2>/dev/null || true)
             LAST_HTTP_CODE=${LAST_HTTP_CODE:-000}
             if echo "$LAST_HTTP_CODE" | grep -qE "^(200|302|404)$"; then
-              echo "CF engine is ready! (HTTP $LAST_HTTP_CODE on attempt $WAIT_COUNT)"
+              echo "CF engine is ready! (HTTP $LAST_HTTP_CODE on attempt $WAIT_COUNT/$MAX_WAIT)"
+              if [ $((WAIT_COUNT * 4)) -gt $((MAX_WAIT * 3)) ]; then
+                echo "::warning::${CFENGINE} became ready at attempt $WAIT_COUNT/$MAX_WAIT (past 3/4 of the ceiling); readiness headroom is shrinking."
+              fi
               READY=true
               break
             fi