Wheels Compatibility Matrix #116

Workflow file for this run

.github/workflows/compat-matrix.yml at ef2624e

	# Compatibility matrix: runs all engines x databases on a weekly schedule
	# and on manual dispatch. Non-blocking — informational only.
	#
	# Each engine x database combination is its own matrix job, so the ~30 legs
	# run in PARALLEL instead of sequentially inside one job per engine. Wall
	# time for a full cycle dropped from ~17 minutes (slowest engine iterating
	# every database in one job, restarting the engine container between legs)
	# to a single leg's duration (~3-5 minutes). Leg isolation is also cleaner:
	# each job gets a fresh engine container + only its own database, so no
	# cross-database state can leak and the per-leg docker restart dance is gone.
	name: Wheels Compatibility Matrix

	on:
	schedule:
	# Weekly: Sunday 02:00 UTC
	- cron: '0 2 * * 0'
	workflow_dispatch:

	env:
	FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true

	jobs:
	tests:
	name: "${{ matrix.cfengine }} + ${{ matrix.database }}"
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	cfengine:
	["lucee6", "lucee7", "adobe2023", "adobe2025", "boxlang"]
	database:
	["mysql", "postgres", "sqlserver", "h2", "cockroachdb", "oracle", "sqlite"]
	exclude:
	# h2 is a Lucee-only datasource
	- cfengine: adobe2023
	database: h2
	- cfengine: adobe2025
	database: h2
	- cfengine: boxlang
	database: h2
	env:
	PORT_lucee6: 60006
	PORT_lucee7: 60007
	PORT_adobe2023: 62023
	PORT_adobe2025: 62025
	PORT_boxlang: 60001
	steps:
	- name: Checkout Repository
	uses: actions/checkout@v5

	- name: Download ojdbc10 for Adobe engines
	if: startsWith(matrix.cfengine, 'adobe')
	run: \|
	mkdir -p ./.engine/${{ matrix.cfengine }}/WEB-INF/lib
	wget -q https://download.oracle.com/otn-pub/otn_software/jdbc/1927/ojdbc10.jar \
	-O ./.engine/${{ matrix.cfengine }}/WEB-INF/lib/ojdbc10.jar

	- name: Start CF engine
	# Retry the build to absorb transient external download failures
	# (GitHub Releases 503/504s for Adoptium JDK/JRE and similar). The
	# build itself is idempotent; only `up -d` runs after success.
	env:
	CFENGINE: ${{ matrix.cfengine }}
	run: \|
	set -e
	MAX_ATTEMPTS=3
	ATTEMPT=1
	until docker compose build --no-cache "$CFENGINE"; do
	if [ $ATTEMPT -ge $MAX_ATTEMPTS ]; then
	echo "::error::docker compose build failed after ${MAX_ATTEMPTS} attempts"
	exit 1
	fi
	echo "::warning::Build attempt ${ATTEMPT} failed — sleeping 30s before retry"
	ATTEMPT=$((ATTEMPT + 1))
	sleep 30
	done
	docker compose up -d "$CFENGINE"

	- name: Start database container
	run: \|
	DB="${{ matrix.database }}"
	case "$DB" in
	mysql\|postgres\|sqlserver\|oracle)
	echo "Starting ${DB}..."
	docker compose up -d "${DB}"
	;;
	cockroachdb)
	# The init sidecar creates the test database/user once the
	# main node is healthy (depends_on: service_healthy).
	echo "Starting cockroachdb + init sidecar..."
	docker compose up -d cockroachdb cockroachdb-init
	;;
	h2\|sqlite)
	echo "$DB requires no external container"
	;;
	esac

	- name: Wait for CF engine to be ready
	env:
	CFENGINE: ${{ matrix.cfengine }}
	run: \|
	PORT_VAR="PORT_${CFENGINE}"
	PORT="${!PORT_VAR}"
	CONTAINER="wheels-${CFENGINE}-1"

	echo "Waiting for ${CFENGINE} on port ${PORT}..."

	# Wait for HTTP response, restarting container if it crashes.
	# Tracks the last HTTP status code so timeout diagnostics can
	# distinguish "no response" (engine didn't bind) from "5xx"
	# (engine bound but app returning errors — e.g. issue #2646).
	MAX_WAIT=60
	WAIT_COUNT=0
	RESTARTS=0
	MAX_RESTARTS=3
	LAST_HTTP_CODE="000"
	while [ "$WAIT_COUNT" -lt "$MAX_WAIT" ]; do
	WAIT_COUNT=$((WAIT_COUNT + 1))

	# Check if container has exited (crashed during startup)
	CONTAINER_STATUS=$(docker inspect --format='{{.State.Status}}' "$CONTAINER" 2>/dev/null \|\| echo "missing")
	if [ "$CONTAINER_STATUS" = "exited" ] \|\| [ "$CONTAINER_STATUS" = "dead" ] \|\| [ "$CONTAINER_STATUS" = "missing" ]; then
	RESTARTS=$((RESTARTS + 1))
	if [ "$RESTARTS" -le "$MAX_RESTARTS" ]; then
	echo "Container $CONTAINER has status '$CONTAINER_STATUS' — restarting (attempt $RESTARTS/$MAX_RESTARTS)..."
	docker compose up -d "${CFENGINE}"
	sleep 10
	continue
	else
	echo "::error::Container $CONTAINER failed to start after $MAX_RESTARTS restart attempts"
	docker logs "$CONTAINER" 2>&1 \| tail -100
	exit 1
	fi
	fi

	# curl with -w "%{http_code}" always prints the code (000 if no
	# response). Don't add a `\|\| echo "000"` fallback — it would
	# concatenate with curl's own 000 output and produce "000000".
	LAST_HTTP_CODE=$(curl -s -o /dev/null --connect-timeout 2 --max-time 5 -w "%{http_code}" "http://localhost:${PORT}/" 2>/dev/null \|\| true)
	LAST_HTTP_CODE=${LAST_HTTP_CODE:-000}
	if echo "$LAST_HTTP_CODE" \| grep -qE "^(200\|302\|404)$"; then
	echo "CF engine is ready! (HTTP $LAST_HTTP_CODE on attempt $WAIT_COUNT)"
	break
	fi

	# Surface partial progress every 10 attempts so logs show
	# whether we're stuck on "no response" or "5xx" early.
	if [ $((WAIT_COUNT % 10)) -eq 0 ]; then
	echo " attempt $WAIT_COUNT/$MAX_WAIT: container=$CONTAINER_STATUS, http=$LAST_HTTP_CODE"
	fi

	if [ "$WAIT_COUNT" -lt "$MAX_WAIT" ]; then
	sleep 5
	fi
	done

	if [ "$WAIT_COUNT" -ge "$MAX_WAIT" ]; then
	echo "::error::CF engine not ready after ${MAX_WAIT} attempts (last HTTP code: $LAST_HTTP_CODE)"
	if [ "$LAST_HTTP_CODE" = "000" ]; then
	echo "::notice::No HTTP response received — engine likely never bound to port $PORT."
	else
	echo "::notice::HTTP $LAST_HTTP_CODE received — engine bound to port $PORT but app is returning errors."
	echo "=== Final response body (first 500 bytes) ==="
	curl -s --max-time 5 "http://localhost:${PORT}/" 2>/dev/null \| head -c 500 \|\| true
	echo
	echo "=== /end response body ==="
	fi
	echo "=== Container logs (stack frames stripped, last 100 lines) ==="
	docker logs "$CONTAINER" 2>&1 \| grep -vE '^\s*at\s\|runwar\.context -[[:space:]]+at[[:space:]]' \| tail -100
	echo "=== /end filtered logs ==="
	echo "=== Container logs (raw, last 200 lines) ==="
	docker logs "$CONTAINER" 2>&1 \| tail -200
	echo "=== /end raw logs ==="
	exit 1
	fi

	- name: Patch Adobe CF serialfilter.txt for Oracle JDBC
	if: (matrix.cfengine == 'adobe2023' \|\| matrix.cfengine == 'adobe2025') && matrix.database == 'oracle'
	run: \|
	docker exec wheels-${{ matrix.cfengine }}-1 sh -c \
	"echo ';oracle.sql.converter.;oracle.sql.;oracle.jdbc.**' >> /wheels-test-suite/.engine/${{ matrix.cfengine }}/WEB-INF/cfusion/lib/serialfilter.txt"
	docker restart wheels-${{ matrix.cfengine }}-1

	# Wait for engine to come back up after restart
	PORT_VAR="PORT_${{ matrix.cfengine }}"
	PORT="${!PORT_VAR}"
	MAX_WAIT=30
	WAIT_COUNT=0
	while [ "$WAIT_COUNT" -lt "$MAX_WAIT" ]; do
	WAIT_COUNT=$((WAIT_COUNT + 1))
	if curl -s -o /dev/null --connect-timeout 2 --max-time 5 -w "%{http_code}" "http://localhost:${PORT}/" \| grep -q "200\\|404\\|302"; then
	echo "CF engine back up after restart"
	break
	fi
	sleep 5
	done

	- name: Install CFPM packages (Adobe 2023/2025)
	if: matrix.cfengine == 'adobe2023' \|\| matrix.cfengine == 'adobe2025'
	run: \|
	MAX_RETRIES=3
	RETRY_COUNT=0

	while [ "$RETRY_COUNT" -lt "$MAX_RETRIES" ]; do
	RETRY_COUNT=$((RETRY_COUNT + 1))
	echo "Attempt $RETRY_COUNT of $MAX_RETRIES: Installing CFPM packages..."

	if docker exec wheels-${{ matrix.cfengine }}-1 box cfpm install image,mail,zip,debugger,caching,mysql,postgresql,sqlserver,oracle; then
	echo "CFPM packages installed successfully"
	exit 0
	else
	echo "CFPM installation failed on attempt $RETRY_COUNT"
	if [ "$RETRY_COUNT" -lt "$MAX_RETRIES" ]; then
	echo "Waiting 10 seconds before retry..."
	sleep 10
	docker exec wheels-${{ matrix.cfengine }}-1 box server restart \|\| true
	sleep 10
	fi
	fi
	done

	echo "Failed to install CFPM packages after $MAX_RETRIES attempts"
	exit 1

	- name: Wait for database to be ready
	run: \|
	DB="${{ matrix.database }}"
	case "$DB" in
	mysql)
	echo "Waiting for MySQL..."
	timeout 60 bash -c 'until docker exec wheels-mysql-1 mysqladmin ping -h localhost -u root -pwheelstestdb --silent 2>/dev/null; do sleep 2; done'
	echo "MySQL is ready"
	;;
	postgres)
	echo "Waiting for PostgreSQL..."
	timeout 60 bash -c 'until docker exec wheels-postgres-1 pg_isready -U wheelstestdb 2>/dev/null; do sleep 2; done'
	echo "PostgreSQL is ready"
	;;
	sqlserver)
	echo "Waiting for SQL Server..."
	timeout 120 bash -c 'until docker exec wheels-sqlserver-1 /opt/mssql-tools18/bin/sqlcmd -S localhost -U SA -P "x!bsT8t60yo0cTVTPq" -Q "SELECT 1" -C 2>/dev/null \| grep -q "1"; do sleep 5; done'
	echo "SQL Server is ready"
	;;
	cockroachdb)
	echo "Waiting for CockroachDB..."
	timeout 60 bash -c 'until docker exec wheels-cockroachdb-1 cockroach sql --insecure -e "SELECT 1" 2>/dev/null; do sleep 2; done'
	echo "CockroachDB is ready"
	echo "Waiting for CockroachDB init to complete..."
	timeout 60 bash -c 'while [ "$(docker inspect --format="{{.State.Status}}" wheels-cockroachdb-init-1 2>/dev/null)" != "exited" ]; do sleep 2; done'
	echo "CockroachDB init complete"
	;;
	oracle)
	echo "Waiting for Oracle to accept connections..."
	MAX_WAIT=60
	WAIT_COUNT=0
	while [ "$WAIT_COUNT" -lt "$MAX_WAIT" ]; do
	WAIT_COUNT=$((WAIT_COUNT + 1))
	if docker exec wheels-oracle-1 sqlplus -S wheelstestdb/wheelstestdb@localhost:1521/wheelstestdb <<< "SELECT 1 FROM DUAL; EXIT;" > /dev/null 2>&1; then
	echo "Oracle is ready! (attempt ${WAIT_COUNT})"
	break
	fi
	echo "Oracle not ready yet (attempt ${WAIT_COUNT}/${MAX_WAIT})..."
	sleep 5
	done
	if [ "$WAIT_COUNT" -ge "$MAX_WAIT" ]; then
	echo "::warning::Oracle may not be fully ready after ${MAX_WAIT} attempts"
	fi
	;;
	h2\|sqlite)
	echo "$DB requires no external container"
	;;
	esac

	- name: Run test suite for ${{ matrix.database }}
	id: run-tests
	run: \|
	PORT_VAR="PORT_${{ matrix.cfengine }}"
	PORT="${!PORT_VAR}"
	BASE_URL="http://localhost:${PORT}/wheels/core/tests"
	DB="${{ matrix.database }}"

	# Databases whose failures are logged but don't block CI.
	# oracle: tracked in #2663 — datasource registration, DBMS_LOCK, and constraint cleanup.
	SOFT_FAIL_DBS="oracle"

	mkdir -p /tmp/test-results
	mkdir -p /tmp/junit-results

	# Warm-up: trigger Wheels onApplicationStart before first test run.
	# The engine readiness check only verifies the web server responds —
	# Wheels app initialization (datasource verification, model scanning)
	# happens on the first real request.
	echo "Warming up Wheels application before first test run..."
	curl -s -o /dev/null --max-time 60 "http://localhost:${PORT}/" \|\| true
	sleep 2

	TEST_URL="${BASE_URL}?db=${DB}&format=json"

	RESULT_FILE="/tmp/test-results/${{ matrix.cfengine }}-${DB}-result.txt"
	JUNIT_FILE="/tmp/junit-results/${{ matrix.cfengine }}-${DB}-junit.xml"

	MAX_RETRIES=3
	RETRY_COUNT=0
	HTTP_CODE="000"

	while [ "$RETRY_COUNT" -lt "$MAX_RETRIES" ]; do
	RETRY_COUNT=$((RETRY_COUNT + 1))
	echo "Test attempt ${RETRY_COUNT} of ${MAX_RETRIES}..."

	HTTP_CODE=$(curl -s -o "$RESULT_FILE" \
	--max-time 900 \
	--write-out "%{http_code}" \
	"$TEST_URL" \|\| echo "000")

	echo "HTTP Code: ${HTTP_CODE}"

	# Stop retrying on success (200) or test failures (417) —
	# 417 means tests ran to completion but some failed, which
	# is a definitive result. Only retry on transient errors.
	if [ "$HTTP_CODE" = "200" ] \|\| [ "$HTTP_CODE" = "417" ]; then
	break
	fi

	if [ "$RETRY_COUNT" -lt "$MAX_RETRIES" ]; then
	echo "Transient error (HTTP ${HTTP_CODE}), waiting 15 seconds before retry..."
	sleep 15
	fi
	done

	# Convert JSON results to JUnit XML locally (avoids a second HTTP
	# request which would re-run the entire test suite — runner.cfm
	# does not cache results between requests)
	if [ -f "$RESULT_FILE" ]; then
	ENGINE="${{ matrix.cfengine }}" DB="${DB}" \
	RESULT_FILE="$RESULT_FILE" JUNIT_FILE="$JUNIT_FILE" \
	python3 -c "
	import json, sys, os
	from xml.etree.ElementTree import Element, SubElement, tostring

	engine = os.environ['ENGINE']
	db = os.environ['DB']
	prefix = f'{engine}/{db}'

	try:
	d = json.load(open(os.environ['RESULT_FILE']))
	except:
	sys.exit(0)

	def safe_str(val, default=''):
	\"\"\"Coerce None/null JSON values to string for XML serialization.\"\"\"
	return str(val) if val is not None else default

	def process_suite(parent_el, suite):
	\"\"\"Recursively process suites (TestBox suites can be nested).\"\"\"
	for sp in suite.get('specStats', []):
	tc = SubElement(parent_el, 'testcase',
	name=safe_str(sp.get('name')),
	classname=f\"{prefix} :: {safe_str(suite.get('name'))}\",
	time=str(sp.get('totalDuration', 0) / 1000))
	if sp.get('status') == 'Failed':
	f = SubElement(tc, 'failure', message=safe_str(sp.get('failMessage')))
	f.text = safe_str(sp.get('failDetail'))
	elif sp.get('status') == 'Error':
	e = SubElement(tc, 'error', message=safe_str(sp.get('failMessage')))
	e.text = safe_str(sp.get('failDetail'))
	elif sp.get('status') == 'Skipped':
	SubElement(tc, 'skipped')
	# Recurse into child suites
	for child in suite.get('suiteStats', []):
	process_suite(parent_el, child)

	root = Element('testsuites',
	name=prefix,
	tests=str(int(d.get('totalSpecs', 0))),
	failures=str(int(d.get('totalFail', 0))),
	errors=str(int(d.get('totalError', 0))),
	time=str(d.get('totalDuration', 0) / 1000))

	for b in d.get('bundleStats', []):
	ts = SubElement(root, 'testsuite',
	name=f\"{prefix} :: {b.get('name', '')}\",
	tests=str(int(b.get('totalSpecs', 0))),
	failures=str(int(b.get('totalFail', 0))),
	errors=str(int(b.get('totalError', 0))),
	time=str(b.get('totalDuration', 0) / 1000))
	for s in b.get('suiteStats', []):
	process_suite(ts, s)

	with open(os.environ['JUNIT_FILE'], 'wb') as f:
	f.write(b'<?xml version=\"1.0\" encoding=\"UTF-8\"?>')
	f.write(tostring(root))
	" \|\| { echo "JUnit conversion failed for ${DB} (non-fatal)"; rm -f "$JUNIT_FILE"; }
	fi

	# Zero-test guard (#3302): a compile-wiped leg returns HTTP 200 with
	# totalSpecs=0 (one bad CFC zeroes the whole directory compile), which
	# previously rendered as a pass. Every engine runs the same core suite
	# (~4,700 specs), so anything below the floor means the suite never
	# actually ran. Revisit the floor if per-DB spec subsets ever ship.
	MIN_SPECS=4000
	TOTAL_SPECS="-1"
	if [ -f "$RESULT_FILE" ] && { [ "$HTTP_CODE" = "200" ] \|\| [ "$HTTP_CODE" = "417" ]; }; then
	TOTAL_SPECS=$(python3 -c "
	import json, sys
	try:
	d = json.load(open('$RESULT_FILE'))
	print(int(d.get('totalSpecs', 0)))
	except:
	print(-1)
	" 2>/dev/null \|\| echo "-1")
	fi

	SPECS_OK=true
	if { [ "$HTTP_CODE" = "200" ] \|\| [ "$HTTP_CODE" = "417" ]; } && [ "$TOTAL_SPECS" -lt "$MIN_SPECS" ]; then
	SPECS_OK=false
	echo "::error::${{ matrix.cfengine }} + ${DB}: HTTP ${HTTP_CODE} but only ${TOTAL_SPECS} testcases reported (floor: ${MIN_SPECS}) — suite likely compile-wiped, treating leg as failed"
	fi

	# Track per-database result
	if [ "$HTTP_CODE" = "200" ] && [ "$SPECS_OK" = true ]; then
	echo "PASSED: ${{ matrix.cfengine }} + ${DB} (${TOTAL_SPECS} testcases)"
	else
	if [ "$HTTP_CODE" = "200" ]; then
	echo "FAILED: ${{ matrix.cfengine }} + ${DB} (HTTP 200 but zero-test guard tripped)"
	else
	echo "FAILED: ${{ matrix.cfengine }} + ${DB} (HTTP ${HTTP_CODE})"
	fi
	if echo "$SOFT_FAIL_DBS" \| grep -qw "$DB"; then
	echo "::warning::${DB} tests failed but marked as soft-fail (non-blocking)"
	else
	echo "One or more database suites failed"
	exit 1
	fi
	fi

	- name: Generate per-leg summary
	if: always()
	run: \|
	DB="${{ matrix.database }}"
	echo "### ${{ matrix.cfengine }} + ${DB} Test Results" >> $GITHUB_STEP_SUMMARY
	echo "" >> $GITHUB_STEP_SUMMARY
	echo "\| Database \| Result \|" >> $GITHUB_STEP_SUMMARY
	echo "\|----------\|--------\|" >> $GITHUB_STEP_SUMMARY

	SOFT_FAIL_DBS="oracle"
	# Keep in sync with MIN_SPECS in the run-tests step (#3302).
	MIN_SPECS=4000
	IS_SOFT_FAIL=false
	if echo "$SOFT_FAIL_DBS" \| grep -qw "$DB"; then
	IS_SOFT_FAIL=true
	fi

	RESULT_FILE="/tmp/test-results/${{ matrix.cfengine }}-${DB}-result.txt"
	if [ -f "$RESULT_FILE" ]; then
	# Check JSON for failures and testcase count (zero-test guard, #3302)
	STATS=$(python3 -c "
	import json, sys
	try:
	d = json.load(open('$RESULT_FILE'))
	print(int(d.get('totalFail', 0) + d.get('totalError', 0)), int(d.get('totalSpecs', 0)))
	except:
	print(-1, -1)
	" 2>/dev/null \|\| echo "-1 -1")
	FAIL_COUNT="${STATS% *}"
	SPEC_COUNT="${STATS#* }"

	if [ "$FAIL_COUNT" = "0" ] && [ "$SPEC_COUNT" -ge "$MIN_SPECS" ]; then
	echo "\| ${DB} \| :white_check_mark: Pass \|" >> $GITHUB_STEP_SUMMARY
	elif [ "$FAIL_COUNT" = "0" ]; then
	echo "\| ${DB} \| :warning: ${SPEC_COUNT} tests (zero-test guard) \|" >> "$GITHUB_STEP_SUMMARY"
	elif [ "$FAIL_COUNT" = "-1" ] && [ "$IS_SOFT_FAIL" = true ]; then
	echo "\| ${DB} \| :warning: Error (soft-fail) \|" >> $GITHUB_STEP_SUMMARY
	elif [ "$FAIL_COUNT" = "-1" ]; then
	echo "\| ${DB} \| :warning: Error \|" >> $GITHUB_STEP_SUMMARY
	elif [ "$IS_SOFT_FAIL" = true ]; then
	echo "\| ${DB} \| :warning: ${FAIL_COUNT} failures (soft-fail) \|" >> $GITHUB_STEP_SUMMARY
	else
	echo "\| ${DB} \| :x: ${FAIL_COUNT} failures \|" >> $GITHUB_STEP_SUMMARY
	fi
	else
	if [ "$IS_SOFT_FAIL" = true ]; then
	echo "\| ${DB} \| :warning: No result (soft-fail) \|" >> $GITHUB_STEP_SUMMARY
	else
	echo "\| ${DB} \| :grey_question: No result \|" >> $GITHUB_STEP_SUMMARY
	fi
	fi

	- name: Debug information
	if: failure()
	run: \|
	echo "=== Docker Container Status ==="
	docker ps -a

	echo -e "\n=== CF Engine Logs ==="
	docker logs $(docker ps -aq -f "name=${{ matrix.cfengine }}") 2>&1 \| tail -100 \|\| echo "Could not get logs"

	echo -e "\n=== Database Container Logs ==="
	docker logs $(docker ps -aq -f "name=${{ matrix.database }}") 2>&1 \| tail -30 \|\| echo "Could not get logs"

	- name: Upload test result artifacts
	if: always()
	uses: actions/upload-artifact@v6
	with:
	name: test-results-${{ matrix.cfengine }}-${{ matrix.database }}
	path: /tmp/test-results/

	- name: Upload JUnit XML artifacts
	if: always()
	uses: actions/upload-artifact@v6
	with:
	name: junit-${{ matrix.cfengine }}-${{ matrix.database }}
	path: /tmp/junit-results/

	#############################################
	# RustCFML (JVM-free engine)
	#############################################
	# Supported engine leg: the release-matrix counterpart of the required
	# PR check in .github/workflows/rustcfml-ci.yml (which runs the same
	# suite on every pull request to develop). The engine is pinned in
	# tools/rustcfml/ENGINE_VERSION (upstream ships multiple releases/day, so
	# tracking latest would make this lane flake on engine churn). Pass criteria
	# is "no NEW failures vs tools/rustcfml/baseline.json" — known residual
	# errors (open upstream issues, listed in the baseline notes) live in the
	# baseline. To bump the pin: update ENGINE_VERSION, run
	# bash tools/rustcfml/run-suite.sh --write-baseline
	# on Linux, and commit both files together.
	rustcfml:
	name: "rustcfml"
	runs-on: ubuntu-latest
	steps:
	- name: Checkout Repository
	uses: actions/checkout@v5

	- name: Read pinned engine version
	id: engine
	run: echo "version=$(tr -d '[:space:]' < tools/rustcfml/ENGINE_VERSION)" >> $GITHUB_OUTPUT

	- name: Cache engine binary
	uses: actions/cache@v4
	with:
	path: ~/.cache/wheels-rustcfml
	key: rustcfml-${{ steps.engine.outputs.version }}-linux-x86_64

	- name: Run core suite against pinned RustCFML
	env:
	GH_TOKEN: ${{ github.token }}
	run: bash tools/rustcfml/run-suite.sh

	#############################################
	# Publish Test Results to PR
	#############################################
	publish-results:
	name: Publish Test Results
	needs: tests
	if: always()
	runs-on: ubuntu-latest
	permissions:
	checks: write
	pull-requests: write
	steps:
	- name: Download JUnit artifacts
	uses: actions/download-artifact@v6
	with:
	pattern: junit-*
	path: junit-results/

	- name: Publish Unit Test Results
	uses: EnricoMi/publish-unit-test-result-action@v2
	with:
	files: junit-results/*/.xml
	check_name: "Wheels Test Results"
	comment_title: "Wheels Test Results"
	# Keep the aggregate check neutral (#3302): oracle soft-fail debt
	# otherwise pins a red "Wheels Test Results" check to whatever SHA
	# the matrix was dispatched on, marking innocent PRs UNSTABLE.
	# Leg pass/fail gating lives in the tests job (OVERALL_STATUS);
	# annotations, PR comments, and artifacts are unaffected by this.
	fail_on: nothing
	report_individual_runs: true
	report_suite_logs: any
	json_file: junit-results/test-results.json
	json_suite_details: true
	json_test_case_results: true
	json_thousands_separator: ","

	#############################################
	# Test Matrix Summary Grid
	#############################################
	test-matrix-summary:
	name: Test Matrix Summary
	needs: tests
	if: always()
	runs-on: ubuntu-latest
	permissions:
	pull-requests: write
	steps:
	- name: Checkout Repository
	uses: actions/checkout@v5

	- name: Download all test result artifacts
	uses: actions/download-artifact@v6
	with:
	pattern: test-results-*
	path: results/

	- name: Generate matrix grid
	id: matrix
	run: \|
	MATRIX_MD="## Wheels Test Matrix"
	MATRIX_MD="${MATRIX_MD}
	"
	MATRIX_MD="${MATRIX_MD}
	\| Engine \| MySQL \| PostgreSQL \| SQL Server \| H2 \| CockroachDB \| Oracle (soft-fail) \| SQLite \|"
	MATRIX_MD="${MATRIX_MD}
	\|--------\|:-----:\|:----------:\|:----------:\|:--:\|:-----------:\|:------------------:\|:------:\|"

	# Keep in sync with SOFT_FAIL_DBS and MIN_SPECS in the tests job (#3302).
	SOFT_FAIL_DBS="oracle"
	MIN_SPECS=4000

	for engine in lucee6 lucee7 adobe2023 adobe2025 boxlang; do
	ROW="\| ${engine} \|"
	for db in mysql postgres sqlserver h2 cockroachdb oracle sqlite; do
	FILE="results/test-results-${engine}-${db}/${engine}-${db}-result.txt"
	IS_SOFT_FAIL=false
	if echo "$SOFT_FAIL_DBS" \| grep -qw "$db"; then
	IS_SOFT_FAIL=true
	fi
	if [ -f "$FILE" ]; then
	STATS=$(python3 -c "
	import json, sys
	try:
	d = json.load(open('$FILE'))
	print(int(d.get('totalFail', 0) + d.get('totalError', 0)), int(d.get('totalSpecs', 0)))
	except:
	print(-1, -1)
	" 2>/dev/null \|\| echo "-1 -1")
	FAIL="${STATS% *}"
	SPECS="${STATS#* }"
	if [ "$FAIL" = "0" ] && [ "$SPECS" -ge "$MIN_SPECS" ]; then
	ROW="${ROW} :white_check_mark: \|"
	elif [ "$FAIL" = "-1" ]; then
	ROW="${ROW} :warning: \|"
	elif [ "$FAIL" = "0" ]; then
	ROW="${ROW} :warning: ${SPECS} tests \|"
	elif [ "$IS_SOFT_FAIL" = true ]; then
	ROW="${ROW} :warning: ${FAIL} \|"
	else
	ROW="${ROW} :x: ${FAIL} \|"
	fi
	else
	ROW="${ROW} -- \|"
	fi
	done
	MATRIX_MD="${MATRIX_MD}
	${ROW}"
	done

	MATRIX_MD="${MATRIX_MD}

	Oracle is soft-fail (non-blocking, tracked in #2663) — :warning: cells in that column never gate the run.
	A ':warning: N tests' cell means the leg reported fewer than ${MIN_SPECS} testcases (suite likely compile-wiped, counted as failed).

	Results for commit ${GITHUB_SHA:0:7}."

	# Write to step summary
	echo "$MATRIX_MD" >> $GITHUB_STEP_SUMMARY

	# Save for PR comment
	echo "$MATRIX_MD" > /tmp/matrix-comment.md

	- name: Post matrix to PR
	if: github.event_name == 'pull_request'
	env:
	GH_TOKEN: ${{ github.token }}
	run: \|
	PR_NUMBER=$(gh pr list --head "${{ github.head_ref \|\| github.ref_name }}" --json number --jq '.[0].number' 2>/dev/null)
	if [ -z "$PR_NUMBER" ]; then
	echo "No PR found, skipping comment"
	exit 0
	fi

	COMMENT_BODY=$(cat /tmp/matrix-comment.md)

	# Look for an existing matrix comment to update
	COMMENT_ID=$(gh api "repos/${{ github.repository }}/issues/${PR_NUMBER}/comments" \
	--jq '.[] \| select(.user.login == "github-actions[bot]" and (.body \| startswith("## Wheels Test Matrix"))) \| .id' \
	2>/dev/null \| head -1)

	if [ -n "$COMMENT_ID" ]; then
	gh api "repos/${{ github.repository }}/issues/comments/${COMMENT_ID}" \
	--method PATCH --field body="$COMMENT_BODY"
	echo "Updated existing comment ${COMMENT_ID}"
	else
	gh pr comment "$PR_NUMBER" --body "$COMMENT_BODY"
	echo "Created new comment on PR #${PR_NUMBER}"
	fi

Navigation Menu

Uh oh!

Uh oh!

Wheels Compatibility Matrix #116

Workflow file

Wheels Compatibility Matrix #116

Uh oh!

Workflow file for this run