fix(cli): wheels test --ci emits GitHub Actions error annotations for failures

changelog.d/3113-cli-test-ci-annotations.fixed.md

@@ -0,0 +1 @@
+- `wheels test --ci` now has an observable effect: it emits GitHub Actions `::error` workflow-command annotations (one per failed or errored spec, with the message encoded to a single line) so failures surface inline in CI logs and PR checks, instead of being byte-identical to a plain run. The flag was previously parsed and threaded through to the runner but never consumed (#3113)

cli/lucli/Module.cfc

@@ -5248,7 +5248,7 @@ component extends="modules.BaseModule" {
 						break;
 					case "simple":
 					default:
-						displayTestResults(result, verboseOutput, resolvedDir);
+						displayTestResults(result, verboseOutput, resolvedDir, ciMode);
 				}
 
 				// Record failure so the command can exit non-zero AFTER the output
@@ -5412,7 +5412,8 @@ component extends="modules.BaseModule" {
 	private void function displayTestResults(
 		required any result,
 		boolean verboseOutput = false,
-		string testDirectory = ""
+		string testDirectory = "",
+		boolean ciMode = false
 	) {
 		if (!isStruct(result)) {
 			out(serializeJSON(result));
@@ -5520,6 +5521,105 @@ component extends="modules.BaseModule" {
 				}
 			}
 		}
+
+		// CI mode (--ci): emit GitHub Actions-style error annotations so each
+		// failure/error surfaces inline in CI logs and PR-check annotations.
+		// testing.mdx documents --ci as tightening output for GitHub Actions
+		// and similar runners; before #3113 the flag was parsed and threaded
+		// through to here but never consumed — byte-identical to a plain run.
+		if (arguments.ciMode) {
+			for (var annotation in $buildCiAnnotations(arguments.result)) {
+				out(annotation);
+			}
+		}
+	}
+
+	/**
+	 * Build GitHub Actions workflow-command annotations (one `::error` line per
+	 * failed or errored spec) from a TestBox result memento. Returns an empty
+	 * array when nothing failed. Pure (no I/O) so it is unit-testable without a
+	 * live server — the `--ci` consumer added for issue #3113.
+	 *
+	 * Format: `::error title=<spec>::<message>`. Message/title are encoded per
+	 * the workflow-command rules (newlines → %0A, % → %25, and `:`/`,` in the
+	 * title) so a multi-line failMessage stays a single annotation line.
+	 */
+	public array function $buildCiAnnotations(required any result) {
+		var annotations = [];
+		if (!isStruct(arguments.result)) {
+			return annotations;
+		}
+
+		// Walk bundle → suite (recursively) → spec, collecting failures. Mirror
+		// the emitTapResults() walker: the closure references itself by name and
+		// appends to a parent struct field (not a bare array) so the mutation is
+		// seen by reference — the established pattern on the CLI's bundled Lucee.
+		var ctx = {failures: []};
+		var walkSuite = function(suite) {
+			for (var spec in (suite.specStats ?: [])) {
+				var status = spec.status ?: "";
+				if (status == "Failed" || status == "Error") {
+					var message = "";
+					if (status == "Failed") {
+						message = spec.failMessage ?: "";
+					} else if (structKeyExists(spec, "error") && isStruct(spec.error)) {
+						message = spec.error.message ?: "";
+					}
+					arrayAppend(ctx.failures, {name: (spec.name ?: "(unnamed spec)"), message: message});
+				}
+			}
+			// Suite-level failure with no specs (compile error, beforeAll threw).
+			if (
+				arrayIsEmpty(suite.specStats ?: [])
+				&& listFindNoCase("Failed,Error", suite.status ?: "")
+			) {
+				arrayAppend(ctx.failures, {
+					name: (suite.name ?: "(unnamed suite)") & " (suite-level)",
+					message: suite.globalException ?: ""
+				});
+			}
+			for (var inner in (suite.suiteStats ?: [])) {
+				walkSuite(inner);
+			}
+		};
+		for (var bundle in (arguments.result.bundleStats ?: [])) {
+			for (var suite in (bundle.suiteStats ?: [])) {
+				walkSuite(suite);
+			}
+		}
+
+		for (var failure in ctx.failures) {
+			arrayAppend(
+				annotations,
+				"::error title=" & $encodeAnnotationProperty(failure.name)
+					& "::" & $encodeAnnotationData(failure.message)
+			);
+		}
+		return annotations;
+	}
+
+	/**
+	 * Encode a GitHub Actions workflow-command data segment (the message after
+	 * `::`). Percent must be escaped first, then carriage returns dropped and
+	 * line feeds collapsed to %0A so the annotation stays one line.
+	 */
+	private string function $encodeAnnotationData(required string value) {
+		var encoded = replace(arguments.value, "%", "%25", "all");
+		encoded = replace(encoded, chr(13), "", "all");
+		encoded = replace(encoded, chr(10), "%0A", "all");
+		return encoded;
+	}
+
+	/**
+	 * Encode a GitHub Actions workflow-command property value (e.g. `title=`).
+	 * Properties additionally escape `:` and `,` so they don't terminate the
+	 * property list.
+	 */
+	private string function $encodeAnnotationProperty(required string value) {
+		var encoded = $encodeAnnotationData(arguments.value);
+		encoded = replace(encoded, ":", "%3A", "all");
+		encoded = replace(encoded, ",", "%2C", "all");
+		return encoded;
 	}
 
 	private void function displaySuite(required struct suite, string indent = "") {

cli/lucli/tests/_fixtures/commands/ModuleOutputCapture.cfc

@@ -32,4 +32,20 @@ component extends="cli.lucli.Module" {
 		return arrayToList(variables.capturedLines, chr(10));
 	}
 
+	/**
+	 * Render a TestBox result struct through the private displayTestResults()
+	 * path and return everything it printed. Lets specs assert the observable
+	 * effect of `--verbose` (per-spec tree) and `--ci` (GitHub Actions
+	 * annotations) without standing up a live test server (issue #3113).
+	 */
+	public string function renderResults(
+		required any result,
+		boolean verboseOutput = false,
+		boolean ciMode = false
+	) {
+		variables.capturedLines = [];
+		displayTestResults(arguments.result, arguments.verboseOutput, "", arguments.ciMode);
+		return capturedOutput();
+	}
+
 }

cli/lucli/tests/specs/commands/TestCommandSpec.cfc

@@ -373,6 +373,121 @@ component extends="wheels.wheelstest.system.BaseSpec" {
 
 		});
 
+		describe("--ci annotation builder ($buildCiAnnotations, issue 3113)", () => {
+
+			it("returns an empty array when nothing failed", () => {
+				var anns = mod.$buildCiAnnotations($passingResult());
+				expect(anns).toBeArray();
+				expect(arrayLen(anns)).toBe(0);
+			});
+
+			it("emits one ::error annotation per failed and errored spec", () => {
+				var anns = mod.$buildCiAnnotations($mixedResult());
+				expect(arrayLen(anns)).toBe(2);
+				var joined = arrayToList(anns, chr(10));
+				expect(joined).toInclude("::error ");
+				expect(joined).toInclude("fails a thing");
+				expect(joined).toInclude("expected true to be false");
+				expect(joined).toInclude("errors a thing");
+				expect(joined).toInclude("boom NPE");
+			});
+
+			it("encodes newlines and percent signs in the annotation message", () => {
+				var result = $failingResult("line1" & chr(10) & "50% off");
+				var anns = mod.$buildCiAnnotations(result);
+				expect(anns[1]).toInclude("line1%0A");
+				expect(anns[1]).toInclude("50%25 off");
+				// The raw newline must not survive — annotations are single-line.
+				expect(anns[1]).notToInclude(chr(10));
+			});
+
+		});
+
+		describe("--ci / --verbose observable output (issue 3113)", () => {
+
+			it("a plain run prints neither a per-spec tree nor CI annotations", () => {
+				var cap = new cli.lucli.tests._fixtures.commands.ModuleOutputCapture(cwd = variables.tempRoot);
+				var printed = cap.renderResults($passingResult(), false, false);
+				expect(printed).notToInclude("[PASS]");
+				expect(printed).notToInclude("::error");
+			});
+
+			it("--verbose prints per-spec PASS lines", () => {
+				var cap = new cli.lucli.tests._fixtures.commands.ModuleOutputCapture(cwd = variables.tempRoot);
+				var printed = cap.renderResults($passingResult(), true, false);
+				expect(printed).toInclude("[PASS]");
+				expect(printed).toInclude("passes a thing");
+			});
+
+			it("--ci prints GitHub Actions error annotations for failures", () => {
+				var cap = new cli.lucli.tests._fixtures.commands.ModuleOutputCapture(cwd = variables.tempRoot);
+				var printed = cap.renderResults($mixedResult(), false, true);
+				expect(printed).toInclude("::error");
+				expect(printed).toInclude("fails a thing");
+			});
+
+		});
+
+	}
+
+	/**
+	 * A TestBox result memento where every spec passed. Shaped like the
+	 * JSONReporter getMemento() the CLI deserializes from /wheels/app|core/tests.
+	 */
+	private struct function $passingResult() {
+		return {
+			totalPass: 1, totalFail: 0, totalError: 0, totalDuration: 12,
+			bundleStats: [{
+				name: "tests.specs.FooSpec",
+				suiteStats: [{
+					name: "Foo feature",
+					status: "Passed",
+					specStats: [{ name: "passes a thing", status: "Passed" }],
+					suiteStats: []
+				}]
+			}]
+		};
+	}
+
+	/**
+	 * A result with one pass, one failure, and one error spec.
+	 */
+	private struct function $mixedResult() {
+		return {
+			totalPass: 1, totalFail: 1, totalError: 1, totalDuration: 34,
+			bundleStats: [{
+				name: "tests.specs.FooSpec",
+				suiteStats: [{
+					name: "Foo feature",
+					status: "Failed",
+					specStats: [
+						{ name: "passes a thing", status: "Passed" },
+						{ name: "fails a thing", status: "Failed", failMessage: "expected true to be false" },
+						{ name: "errors a thing", status: "Error", error: { message: "boom NPE" } }
+					],
+					suiteStats: []
+				}]
+			}]
+		};
+	}
+
+	/**
+	 * A result with a single failure carrying the given fail message — used
+	 * to exercise annotation message encoding.
+	 */
+	private struct function $failingResult(required string failMessage) {
+		return {
+			totalPass: 0, totalFail: 1, totalError: 0, totalDuration: 5,
+			bundleStats: [{
+				name: "tests.specs.FooSpec",
+				suiteStats: [{
+					name: "Foo feature",
+					status: "Failed",
+					specStats: [{ name: "fails a thing", status: "Failed", failMessage: arguments.failMessage }],
+					suiteStats: []
+				}]
+			}]
+		};
 	}
 
 	/**

web/sites/guides/src/content/docs/v4-0-0/command-line-tools/wheels-commands/testing.mdx

@@ -59,7 +59,7 @@ A bare positional argument is treated as the filter directory — `wheels test m
 | `--db=<engine>` | Database engine for `--core` matrix runs only. Ignored for app tests (with a warning) — see [below](#testing-against-different-engines). |
 | `--reporter=<name>` | `simple` (default, colourful), `json` (raw runner JSON), `tap` (TAP v13 for CI consumers). |
 | `--verbose`, `-v` | Accepted but currently inert — output is identical to a plain run; no per-spec output is printed for passing specs. Wiring tracked in [#3113](https://github.com/wheels-dev/wheels/issues/3113). |
-| `--ci` | Accepted but currently inert — output is byte-identical to a plain run, and exit codes are already non-zero on failure without it. Intended to tighten output for GitHub Actions and similar runners; tracked in [#3113](https://github.com/wheels-dev/wheels/issues/3113). |
+| `--ci` | CI mode: emits one GitHub Actions `::error` workflow-command annotation per failed or errored spec, so failures surface inline in CI logs and PR-check panels. Exit code is non-zero on failure regardless. |
 | `--core` | Run framework self-tests (`vendor/wheels/tests/specs/`) instead of your app suite. App tests are the default; `--core` is the explicit opt-in. |
 | `--no-test-db` | Disable the auto-swap to `<datasource>_test`. App tests run against your dev datasource, with whatever data is already in it. |
 | `--base-path=<path>` | URL prefix the app is mounted under (e.g. `/myapp`). Auto-derived from `WHEELS_SUBPATH` or `set(subpath=...)` in `config/settings.cfm` when omitted. Leave unset for root-mounted apps (the default). |
@@ -155,8 +155,10 @@ wheels test
 # Narrow to one area while iterating
 wheels test --filter=models
 
-# CI run — exit codes are already firm without extra flags; for machine-readable
-# output use a reporter (--ci itself is inert today, #3113)
+# CI run on GitHub Actions — --ci adds one ::error annotation per failed or
+# errored spec (exit codes are firm with or without it); for machine-readable
+# output use a reporter
+wheels test --ci
 wheels test --reporter=tap
 
 # First-time browser setup, then exercise the browser specs

web/sites/guides/src/content/docs/v4-0-0/testing/ci-integration.mdx

@@ -78,8 +78,8 @@ The `wheels test` command accepts a `--reporter=<name>` flag. The CLI always req
 | `--reporter=simple` (default) | Human-readable summary: `N passed (Xs)` on green, plus failure details on red |
 | `--reporter=json` | Emits the raw JSON result document — pipe it to `jq` or a post-processor |
 | `--reporter=tap` | Emits TAP version 13 (`1..N`, `ok` / `not ok` lines) for TAP-consuming CI tooling |
-| `--ci` | Accepted for forward-compatibility — currently changes nothing; every run already exits non-zero on failure |
-| `--verbose` / `-v` | Adds the full bundle/suite/spec tree to the output |
+| `--ci` | Emits one GitHub Actions `::error` workflow-command annotation per failed or errored spec, so failures appear inline in CI logs and PR-check panels. Exit code is non-zero on failure regardless. |
+| `--verbose` / `-v` | Accepted but currently inert — output is identical to a plain run; the per-spec tree wiring is tracked in [#3113](https://github.com/wheels-dev/wheels/issues/3113) |
 
 For machine-readable results you can also call the test runner URL directly and post-process the JSON. That is exactly what `tools/ci/run-tests.sh` does in this repo: it `curl`s `/wheels/core/tests?db=sqlite&format=json`, parses the totals in Python, emits a JUnit XML file that `actions/upload-artifact` ingests for the GitHub summary, and fails the build when the payload reports a rejected `directory=` scope or a 0-bundle discovery.
 

web/sites/guides/src/content/docs/v4-0-0/testing/running-tests-locally.mdx

@@ -55,7 +55,7 @@ wheels test --reporter=tap
 # Target a specific test database — only meaningful with --core
 wheels test --core --db=mysql
 
-# Accepted for forward-compat — every run already exits non-zero on failures
+# CI mode: emits GitHub Actions ::error annotations per failed/errored spec
 wheels test --ci
 ```