Merge pull request #23 from superdurable/codex/real-integration-tests-only

Keep only real API integration tests

Author: longquanzheng

.github/workflows/update-dex-dependencies.yml

@@ -84,9 +84,9 @@ jobs:
             README.md \
             go.mod \
             go.sum \
-            internal/templatecontract/contract_test.go
+            tools/checkcontract/main.go
           git diff --cached --check
-          go test ./internal/templatecontract
+          make check-contracts
       - name: Create or update dependency pull request
         if: steps.update.outputs.changed == 'true'
         shell: bash

.superverse/template.json

@@ -1,7 +1,7 @@
 {
   "schemaVersion": 1,
   "buildProfile": "go-react-v1",
-  "templateVersion": "1.8.1",
+  "templateVersion": "1.8.2",
   "minimumSandboxImageContractRevision": 3,
   "openapiSpec": "openapi/openapi.yaml",
   "applicationManifest": "dex-app.yaml",
@@ -11,7 +11,7 @@
     "generate": "make generate",
     "checkFdgV2": "make check-fdg-v2",
     "releaseArtifacts": "make superverse-release-artifacts",
-    "testUnit": "make test-unit",
+    "checkContracts": "make check-contracts",
     "testIntegration": "make test-integration",
     "testE2E": "make test-e2e",
     "build": "make build",

AGENTS.md

@@ -36,9 +36,9 @@ retain either stem, and compound names such as `AppRuntime`, `RuntimeManager`,
 or `NormalizeInput`. Renaming must update the package path, declarations,
 imports, callers, and tests together; do not retain compatibility aliases for
 repository-owned names. `internal/worker` owns the Dex Worker, Client,
-and blob cache lifecycle. The Go naming contract in `internal/templatecontract`
-runs with `make test-unit` and `make check`; do not bypass it or add exceptions
-for new repository-owned names.
+and blob cache lifecycle. The static Go naming check in `tools/checkcontract`
+runs with `make check-contracts` and `make check`; do not bypass it or add
+exceptions for new repository-owned names.
 
 The Dex Flow has stable Step, Attribute, Channel, Timer, and RPC identities.
 Keep external effects in `Execute`; `WaitForApproval.WaitFor` only declares the
@@ -58,7 +58,7 @@ If `make check` fails or cannot run, report `blocked=true`. Do not weaken, skip,
 or delete a failing check.
 
 Stable commands are `make bootstrap`, `make generate`, `make check-fdg-v2`,
-`make superverse-release-artifacts`, `make test-unit`,
+`make check-contracts`, `make superverse-release-artifacts`,
 `make test-integration`, `make test-e2e`, `make build`, `make dev`, and
 `make check`.
 
@@ -97,12 +97,20 @@ Dex Skills discovers the release and opens its own baseline pull request. Once
 the matching Dex Skills release exists, Superverse advances both immutable
 release pins in one pull request.
 
-Use Vitest mocks of the generated client for isolated loading, failure, and
-terminal UI states. When a browser-only edge case cannot be reached
-economically, use test-local Playwright request interception. Do not add an
-application mock server, a second business state machine, or user-visible Mock
-Controls. Mock evidence never replaces real Dex integration and E2E tests.
+Only integration tests that call real dependency APIs are permitted. Never add
+unit tests, isolated component tests, or mock integration tests in any language.
+Go tests must use the `integration` build tag and a real Dex Server. Playwright
+E2E tests must drive the production application backed by real dependencies.
+Do not use mocks, fakes, stubs, simulated providers, or intercepted/fulfilled
+API responses to replace dependencies, including for failure or edge cases.
+Exercise those cases through real dependency APIs or report the coverage gap.
+An integration label or build tag alone does not qualify a test: review its
+actual API calls. Dependency startup failures must fail verification, never
+skip a test or fall back to a mock. Do not restore unit-test targets, frameworks,
+fixtures, or dependencies. `make check-contracts` statically checks repository
+contracts, naming, test placement, and known mocking APIs; it is not a test suite
+and does not replace real integration or E2E evidence.
 
 When structure, commands, or required tooling changes, update this file,
-`.superverse/template.json`, `README.md`, and contract tests together. Do not
-maintain a separate static repository map.
+`.superverse/template.json`, `README.md`, and static contract checks together.
+Do not maintain a separate static repository map.

Makefile

@@ -1,6 +1,6 @@
 SHELL := /usr/bin/env bash
 
-.PHONY: bootstrap generate check-fdg-v2 superverse-release-artifacts test-unit test-integration test-e2e build dev check
+.PHONY: bootstrap generate check-fdg-v2 check-contracts superverse-release-artifacts test-integration test-e2e build dev check
 bootstrap:
 	go mod download
 	go -C tools/openapi mod download
@@ -12,27 +12,22 @@ check-fdg-v2:
 superverse-release-artifacts:
 	@test -n "$(SUPERVERSE_RELEASE_ARTIFACT_DIR)" || { echo "SUPERVERSE_RELEASE_ARTIFACT_DIR is required" >&2; exit 1; }
 	python3 ./scripts/generate-release-artifacts.py --output-directory "$(SUPERVERSE_RELEASE_ARTIFACT_DIR)"
-test-unit: generate
-	go test ./...
-	python3 -m unittest scripts/test_check_template_version.py scripts/test_generate_openapi.py scripts/test_generate_release_artifacts.py
-	npm --prefix web test
-test-integration: generate
-	./scripts/with-dex.sh go test -tags=integration ./...
-test-e2e: generate
+check-contracts:
+	go run ./tools/checkcontract
+test-integration: generate check-contracts
+	./scripts/with-dex.sh go test -count=1 -tags=integration ./...
+test-e2e: generate check-contracts
 	./scripts/with-dex.sh ./scripts/run-e2e.sh
 build: generate
 	npm --prefix web run build
 	go build -o bin/basic-process ./cmd/server
 dev: generate
 	./scripts/with-dex.sh bash -c 'npm --prefix web run build && go run ./cmd/server'
-check: bootstrap generate check-fdg-v2
+check: bootstrap generate check-fdg-v2 check-contracts
 	@test -z "$$(gofmt -l $$(find . -name '*.go' -not -path './upstream-dex/*'))" || { gofmt -d $$(gofmt -l $$(find . -name '*.go' -not -path './upstream-dex/*')); exit 1; }
 	go mod tidy -diff
 	go vet ./...
-	go test ./...
-	python3 -m unittest scripts/test_check_template_version.py scripts/test_generate_openapi.py scripts/test_generate_release_artifacts.py
-	npm --prefix web test
-	./scripts/with-dex.sh go test -tags=integration ./...
+	./scripts/with-dex.sh go test -count=1 -tags=integration ./...
 	./scripts/with-dex.sh ./scripts/run-e2e.sh
 	npm --prefix web run build
 	go build -o bin/basic-process ./cmd/server

README.md

@@ -35,7 +35,8 @@ Repository-owned names must describe concrete responsibilities. The
 case-insensitive stems `runtime` and `normaliz` are prohibited in names,
 including package paths, types, aliases, receivers, and test helpers. Generated,
 third-party, framework-mandated, and immutable legacy references are exempt.
-Go naming contract tests enforce this rule in `make test-unit` and `make check`.
+Static checks in `tools/checkcontract` enforce this rule in
+`make check-contracts` and `make check`.
 
 Applications that do not need a custom process UI keep only a non-business
 Hello World page and the Go/OpenAPI/React generation skeleton. Dex Web remains
@@ -108,21 +109,30 @@ the platform broker.
 ## Verification
 
 ```bash
-make test-unit
+make check-contracts
 make test-integration
 make test-e2e
 make check
 ```
 
 Integration tests start a real Dex Server with `dexcli dev`. Playwright drives
 the production UI and uses `dexcli flow skip-timer` to exercise the reminder
-branch without waiting fifteen minutes. Every poll has a deadline.
-
-Vitest mocks the generated client for isolated loading, failure, and terminal
-UI states. A test may use Playwright request interception for a browser-only
-edge case, but the template does not ship a mock API, a second business state
-machine, or user-visible Mock Controls. Mock evidence does not prove durable
-execution behavior.
+branch without waiting fifteen minutes. Every poll has a deadline. Go
+integration tests disable result caching so each run calls the real Dex APIs.
+
+Only integration tests that call real dependency APIs are permitted, including
+the Playwright E2E journey against the production application. Unit tests,
+isolated component tests, mock integration tests, fakes, stubs, and Playwright
+API-response interception are prohibited, including for failure and edge cases.
+Exercise those cases through real dependency APIs or report the coverage gap.
+An integration label alone is insufficient. Missing dependencies fail the
+check; tests must never skip or fall back to mocks. Do not add unit-test
+frameworks, fixtures, or dependencies.
+
+`make check-contracts` validates the template manifest, Go naming, test
+placement, and known mocking APIs through static source inspection. It is
+separate from API tests and does not prove dependency behavior. The dependency
+updater keeps its release assertions in `tools/checkcontract/main.go`.
 
 `make check-fdg-v2` validates `internal/process/flow.go` with rendering schema
 2.0 and requires a diagnostic-free graph with `valid: true`. The required