From 51df6e1ac56170cbd7a08904ec1bbe6d5049deaa Mon Sep 17 00:00:00 2001 From: Peter Amiri Date: Tue, 4 Aug 2026 15:25:37 -0700 Subject: [PATCH 1/2] ci: compat-matrix safe-slice hardening for #3302 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three workflow-plumbing fixes in compat-matrix.yml, shippable before the continue-on-error flip (which stays gated on PR #3365 merging plus one green dispatch on develop): - run-tests: per-leg zero-test guard — an HTTP 200/417 leg reporting totalSpecs below 4000 (suite runs ~4,700) now emits ::error:: and sets the leg fail flag, so a compile-wiped leg fails loudly instead of rendering as a pass. SOFT_FAIL_DBS is respected. - publish-results: fail_on: nothing on publish-unit-test-result-action, so oracle soft-fail debt stops pinning a red aggregate 'Wheels Test Results' check to innocent dispatch SHAs. Annotations, PR comments and artifacts are unchanged; leg gating stays in the tests job. - test-matrix-summary: zero-test legs render as ':warning: N tests' instead of a checkmark, soft-fail DB failures render as :warning:, and the Oracle column is annotated as soft-fail with a footnote. The rustcfml job is untouched (intentionally informational). Refs #3302 Signed-off-by: Peter Amiri --- .github/workflows/compat-matrix.yml | 71 +++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 10 deletions(-) diff --git a/.github/workflows/compat-matrix.yml b/.github/workflows/compat-matrix.yml index 5d30ee1fa..e38c662c2 100644 --- a/.github/workflows/compat-matrix.yml +++ b/.github/workflows/compat-matrix.yml @@ -439,12 +439,40 @@ jobs: " || { echo "JUnit conversion failed for ${db} (non-fatal)"; rm -f "$JUNIT_FILE"; } fi + # Zero-test guard (#3302): a compile-wiped leg returns HTTP 200 with + # totalSpecs=0 (one bad CFC zeroes the whole directory compile), which + # previously rendered as a pass. Every engine runs the same core suite + # (~4,700 specs), so anything below the floor means the suite never + # actually ran. Revisit the floor if per-DB spec subsets ever ship. + MIN_SPECS=4000 + TOTAL_SPECS="-1" + if [ -f "$RESULT_FILE" ] && { [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "417" ]; }; then + TOTAL_SPECS=$(python3 -c " + import json, sys + try: + d = json.load(open('$RESULT_FILE')) + print(int(d.get('totalSpecs', 0))) + except: + print(-1) + " 2>/dev/null || echo "-1") + fi + + SPECS_OK=true + if { [ "$HTTP_CODE" = "200" ] || [ "$HTTP_CODE" = "417" ]; } && [ "$TOTAL_SPECS" -lt "$MIN_SPECS" ]; then + SPECS_OK=false + echo "::error::${{ matrix.cfengine }} + ${db}: HTTP ${HTTP_CODE} but only ${TOTAL_SPECS} testcases reported (floor: ${MIN_SPECS}) — suite likely compile-wiped, treating leg as failed" + fi + # Track per-database result - if [ "$HTTP_CODE" = "200" ]; then - echo "PASSED: ${{ matrix.cfengine }} + ${db}" + if [ "$HTTP_CODE" = "200" ] && [ "$SPECS_OK" = true ]; then + echo "PASSED: ${{ matrix.cfengine }} + ${db} (${TOTAL_SPECS} testcases)" DB_STATUS="pass" else - echo "FAILED: ${{ matrix.cfengine }} + ${db} (HTTP ${HTTP_CODE})" + if [ "$HTTP_CODE" = "200" ]; then + echo "FAILED: ${{ matrix.cfengine }} + ${db} (HTTP 200 but zero-test guard tripped)" + else + echo "FAILED: ${{ matrix.cfengine }} + ${db} (HTTP ${HTTP_CODE})" + fi DB_STATUS="fail" if echo "$SOFT_FAIL_DBS" | grep -qw "$db"; then echo "::warning::${db} tests failed but marked as soft-fail (non-blocking)" @@ -613,6 +641,12 @@ jobs: files: junit-results/**/*.xml check_name: "Wheels Test Results" comment_title: "Wheels Test Results" + # Keep the aggregate check neutral (#3302): oracle soft-fail debt + # otherwise pins a red "Wheels Test Results" check to whatever SHA + # the matrix was dispatched on, marking innocent PRs UNSTABLE. + # Leg pass/fail gating lives in the tests job (OVERALL_STATUS); + # annotations, PR comments, and artifacts are unaffected by this. + fail_on: nothing report_individual_runs: true report_suite_logs: any json_file: junit-results/test-results.json @@ -647,27 +681,41 @@ jobs: MATRIX_MD="${MATRIX_MD} " MATRIX_MD="${MATRIX_MD} - | Engine | MySQL | PostgreSQL | SQL Server | H2 | CockroachDB | Oracle | SQLite |" + | Engine | MySQL | PostgreSQL | SQL Server | H2 | CockroachDB | Oracle (soft-fail) | SQLite |" MATRIX_MD="${MATRIX_MD} - |--------|:-----:|:----------:|:----------:|:--:|:-----------:|:------:|:------:|" + |--------|:-----:|:----------:|:----------:|:--:|:-----------:|:------------------:|:------:|" + + # Keep in sync with SOFT_FAIL_DBS and MIN_SPECS in the tests job (#3302). + SOFT_FAIL_DBS="oracle" + MIN_SPECS=4000 for engine in lucee6 lucee7 adobe2023 adobe2025 boxlang; do ROW="| **${engine}** |" for db in mysql postgres sqlserver h2 cockroachdb oracle sqlite; do FILE="results/test-results-${engine}/${engine}-${db}-result.txt" + IS_SOFT_FAIL=false + if echo "$SOFT_FAIL_DBS" | grep -qw "$db"; then + IS_SOFT_FAIL=true + fi if [ -f "$FILE" ]; then - FAIL=$(python3 -c " + STATS=$(python3 -c " import json, sys try: d = json.load(open('$FILE')) - print(int(d.get('totalFail', 0) + d.get('totalError', 0))) + print(int(d.get('totalFail', 0) + d.get('totalError', 0)), int(d.get('totalSpecs', 0))) except: - print(-1) - " 2>/dev/null || echo "-1") - if [ "$FAIL" = "0" ]; then + print(-1, -1) + " 2>/dev/null || echo "-1 -1") + FAIL="${STATS% *}" + SPECS="${STATS#* }" + if [ "$FAIL" = "0" ] && [ "$SPECS" -ge "$MIN_SPECS" ]; then ROW="${ROW} :white_check_mark: |" elif [ "$FAIL" = "-1" ]; then ROW="${ROW} :warning: |" + elif [ "$FAIL" = "0" ]; then + ROW="${ROW} :warning: ${SPECS} tests |" + elif [ "$IS_SOFT_FAIL" = true ]; then + ROW="${ROW} :warning: ${FAIL} |" else ROW="${ROW} :x: ${FAIL} |" fi @@ -681,6 +729,9 @@ jobs: MATRIX_MD="${MATRIX_MD} + *Oracle is soft-fail (non-blocking, tracked in #2663) — :warning: cells in that column never gate the run.* + *A ':warning: N tests' cell means the leg reported fewer than ${MIN_SPECS} testcases (suite likely compile-wiped, counted as failed).* + *Results for commit ${GITHUB_SHA:0:7}.*" # Write to step summary From 195a46562d9844655284ba4dd401a5ef8f7dae7f Mon Sep 17 00:00:00 2001 From: Peter Amiri Date: Tue, 4 Aug 2026 15:32:02 -0700 Subject: [PATCH 2/2] ci: align per-engine summary with the zero-test guard Review follow-up for the #3302 safe slice: the per-engine step summary still rendered a compile-wiped leg (0 failures, 0 testcases) as a pass while the run-tests guard in the same job failed it with ::error::. Read totalSpecs alongside the failure count and render sub-floor legs as ':warning: N tests (zero-test guard)', mirroring the matrix grid. Refs #3302 Signed-off-by: Peter Amiri --- .github/workflows/compat-matrix.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/compat-matrix.yml b/.github/workflows/compat-matrix.yml index e38c662c2..e3a86589f 100644 --- a/.github/workflows/compat-matrix.yml +++ b/.github/workflows/compat-matrix.yml @@ -514,6 +514,8 @@ jobs: echo "|----------|--------|" >> $GITHUB_STEP_SUMMARY SOFT_FAIL_DBS="oracle" + # Keep in sync with MIN_SPECS in the run-tests step (#3302). + MIN_SPECS=4000 IFS=',' read -ra DBS <<< "${{ steps.db-list.outputs.databases }}" for db in "${DBS[@]}"; do RESULT_FILE="/tmp/test-results/${{ matrix.cfengine }}-${db}-result.txt" @@ -522,18 +524,22 @@ jobs: IS_SOFT_FAIL=true fi if [ -f "$RESULT_FILE" ]; then - # Check JSON for failures - FAIL_COUNT=$(python3 -c " + # Check JSON for failures and testcase count (zero-test guard, #3302) + STATS=$(python3 -c " import json, sys try: d = json.load(open('$RESULT_FILE')) - print(d.get('totalFail', 0) + d.get('totalError', 0)) + print(int(d.get('totalFail', 0) + d.get('totalError', 0)), int(d.get('totalSpecs', 0))) except: - print(-1) - " 2>/dev/null || echo "-1") + print(-1, -1) + " 2>/dev/null || echo "-1 -1") + FAIL_COUNT="${STATS% *}" + SPEC_COUNT="${STATS#* }" - if [ "$FAIL_COUNT" = "0" ]; then + if [ "$FAIL_COUNT" = "0" ] && [ "$SPEC_COUNT" -ge "$MIN_SPECS" ]; then echo "| ${db} | :white_check_mark: Pass |" >> $GITHUB_STEP_SUMMARY + elif [ "$FAIL_COUNT" = "0" ]; then + echo "| ${db} | :warning: ${SPEC_COUNT} tests (zero-test guard) |" >> "$GITHUB_STEP_SUMMARY" elif [ "$FAIL_COUNT" = "-1" ] && [ "$IS_SOFT_FAIL" = true ]; then echo "| ${db} | :warning: Error (soft-fail) |" >> $GITHUB_STEP_SUMMARY elif [ "$FAIL_COUNT" = "-1" ]; then