Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
68 changes: 68 additions & 0 deletions scripts/qa-result-gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -1088,6 +1088,50 @@ classification_ids_cover_source_fail_ids() {
return 0
}

# Detect a clean phase-level terminal baseline (issue #655): the phase's
# authoritative verification resolves on disk, is a PASS result, lists zero FAIL
# rows, and carries zero pre-existing/known issues. This identifies a no-FAIL
# disposition phase (diagnosis / documentation) that has genuinely nothing left
# to remediate, distinguishing it from a continuation round that simply lost its
# source verification (no phase verification on disk) or a phase that still
# carries unresolved pre-existing issues.
phase_baseline_is_clean() {
local phase_dir="${1:-}" phase_v result preexist
[ -n "$phase_dir" ] || return 1
phase_v=$(bash "$SCRIPT_DIR/resolve-verification-path.sh" phase "$phase_dir" 2>/dev/null || true)
[ -n "$phase_v" ] && [ -r "$phase_v" ] || return 1
# Normalize like the gate's authoritative RESULT parser: case-insensitive,
# with a fallback to the legacy `status:` frontmatter field when `result:` is
# absent, so the same on-disk verification is judged consistently here.
result=$(extract_frontmatter_scalar_value "$phase_v" result)
[ -n "$result" ] || result=$(extract_frontmatter_scalar_value "$phase_v" status)
result=$(printf '%s' "$result" | tr '[:lower:]' '[:upper:]')
[ "$result" = "PASS" ] || return 1
[ "$(count_fail_rows_in_verification "$phase_v")" -eq 0 ] 2>/dev/null || return 1
preexist=$(awk '
/^## Pre-existing Issues/ { found=1; header_done=0; next }
found && /^## / { exit }
found && /^\|/ {
# Skip GitHub-style alignment separator rows (dashes, optional colons).
if ($0 ~ /^\|[[:space:]:|-]+$/) next
# The first non-separator table row is the column header, whatever its
# column titles are; every subsequent row is a counted data row.
if (!header_done) { header_done=1; next }
# Skip placeholder rows (e.g. "| None | - | - |") so a clean phase that
# renders an explicit empty pre-existing table still reads as clean.
split($0, _cells, "|")
_c = _cells[2]
gsub(/^[[:space:]]+|[[:space:]]+$/, "", _c)
_lc = tolower(_c)
if (_lc == "" || _lc == "none" || _lc == "n/a" || _lc == "na" || _lc ~ /^none[.,; ]/ || _lc ~ /^no .*(issue|pre-existing)/) next
count++
}
END { print count+0 }
' "$phase_v" 2>/dev/null)
[ "${preexist:-0}" -eq 0 ] 2>/dev/null || return 1
return 0
}

extract_frontmatter_json_object_array() {
local file_path="${1:-}"
local key_name="${2:-}"
Expand Down Expand Up @@ -1980,6 +2024,30 @@ if [ "$IN_REMEDIATION" = "true" ] && [ "$SUMMARY_SCOPE_DIR" != "$PHASE_DIR" ]; t
ROUND_SOURCE_VERIFICATION_MISSING="false"
fi

# No-FAIL documentation / plan-amendment disposition (issue #655 sub-defect B):
# a round that classifies nothing (zero fail-classifications), carries no
# known issues, recorded only planning-metadata files, and has no source
# verification path has, by design, no FAIL source to anchor against — there
# is nothing to remediate. Exempt it from the source-missing check at ANY
# stage so the verify stage does not deadlock and force an unbounded re-run
# (the round could otherwise escape only by manually advancing verify -> done).
#
# The discriminator is the RECORDED files (not METADATA_ONLY_ROUND, which is
# corroboration-dependent and reads non-metadata edits as metadata-only when no
# git evidence is available) plus a clean phase-level baseline. A round that
# carries fail-classifications or known issues, recorded any non-metadata file,
# or whose phase verification is absent / not-PASS / carries FAIL rows or
# pre-existing issues does NOT satisfy this and still fails closed.
if [ "${ROUND_INPUT_MODE:-none}" = "none" ] \
&& [ -z "$SOURCE_VERIFICATION_PATH" ] \
&& [ "${ROUND_CLASSIFICATION_TYPE_COUNT:-0}" -eq 0 ] 2>/dev/null \
&& [ "${KNOWN_ISSUES_COUNT:-0}" -eq 0 ] 2>/dev/null \
&& [ -n "$ROUND_RECORDED_STRUCTURAL_PATHS" ] \
&& ! paths_include_non_metadata "$PHASE_DIR" <<< "$ROUND_RECORDED_STRUCTURAL_PATHS" \
&& phase_baseline_is_clean "$PHASE_DIR"; then
ROUND_SOURCE_VERIFICATION_MISSING="false"
fi

if paths_include_process_exception_evidence "$PHASE_DIR" <<< "$ROUND_ALL_RECORDED_PATHS"; then
ROUND_PROCESS_EXCEPTION_EVIDENCE_VALID=true
elif [ "$METADATA_ONLY_ROUND" = "true" ] \
Expand Down
189 changes: 187 additions & 2 deletions tests/qa-result-gate.bats
Original file line number Diff line number Diff line change
Expand Up @@ -2231,7 +2231,15 @@ VERIF
[[ "$output" == *"qa_gate_routing=PROCEED_TO_UAT"* ]]
}

@test "metadata-only round with missing fail_classifications → REMEDIATION_REQUIRED" {
# Issue #655 (sub-defect B): a no-classification, metadata-only round on a phase
# whose authoritative verification is a clean PASS (0 FAIL, 0 pre-existing) has
# genuinely nothing to remediate, so it must terminate (PROCEED_TO_UAT) rather
# than deadlock at the verify stage on source_verification_missing. A historical
# phase-root SUMMARY deviation does not force remediation — phase-root deviations
# are already treated as historical during remediation (they are not counted in
# the round-scoped deviation override). Previously this routed REMEDIATION_REQUIRED
# via the stage-asymmetric source-missing check; that was the #655 loop.
@test "no-classification metadata-only round on a clean-baseline phase → PROCEED_TO_UAT (issue #655)" {
create_verif "write-verification.sh" "PASS"
create_summary_with_yaml_deviations "01-01" "Historical orchestration issue"

Expand All @@ -2244,7 +2252,7 @@ VERIF
cat > "$PHASE_DIR/remediation/qa/round-01/R01-PLAN.md" <<'PLAN'
---
round: 01
title: Missing classifications should fail closed
title: No-FAIL documentation disposition
---
PLAN
cat > "$PHASE_DIR/remediation/qa/round-01/R01-VERIFICATION.md" <<'VERIF'
Expand All @@ -2263,9 +2271,186 @@ VERIF
run bash "$SCRIPT" "$PHASE_DIR"

[ "$status" -eq 0 ]
[[ "$output" != *"qa_gate_source_verification_missing=true"* ]]
[[ "$output" == *"qa_gate_routing=PROCEED_TO_UAT"* ]]
}

# Issue #655 (sub-defect B) — fail-closed companions. The source-missing
# exemption must fire ONLY when every guard passes; the tests below each satisfy
# the earlier guards (input_mode=none, 0 classifications, 0 known issues) and
# then trip exactly one remaining guard, so each branch is exercised live:
# - this test: a recorded NON-metadata edit (claims real work, no source)
# - the three tests after it: the reachable false-return branches of
# phase_baseline_is_clean (absent phase verification, non-PASS result,
# pre-existing issues). Its count_fail==0 guard is defense-in-depth and is
# not independently reachable through the gate: a phase verification with
# FAIL rows makes qa-remediation-state derive input_mode=verification, so the
# exemption's input_mode=none precondition already excludes that case.
@test "no-classification round recording a non-metadata edit still fails closed (issue #655)" {
create_verif "write-verification.sh" "PASS"

mkdir -p "$PHASE_DIR/remediation/qa/round-01"
printf 'stage=verify\nround=01\n' > "$PHASE_DIR/remediation/qa/.qa-remediation-stage"

create_round_summary_with_files "$PHASE_DIR/remediation/qa/round-01" "01" \
' - "src/Feature.swift"'

cat > "$PHASE_DIR/remediation/qa/round-01/R01-PLAN.md" <<'PLAN'
---
round: 01
title: Non-metadata edit without a source must fail closed
---
PLAN
cat > "$PHASE_DIR/remediation/qa/round-01/R01-VERIFICATION.md" <<'VERIF'
---
writer: write-verification.sh
result: PASS
plans_verified:
- R01
---
## Checks
| ID | Category | Description | Status | Evidence |
|----|----------|-------------|--------|----------|
| MH-01 | must_have | Code edited | PASS | Done |
VERIF

run bash "$SCRIPT" "$PHASE_DIR"

[ "$status" -eq 0 ]
[[ "$output" == *"qa_gate_source_verification_missing=true"* ]]
[[ "$output" == *"qa_gate_routing=REMEDIATION_REQUIRED"* ]]
}

# Helper: a metadata-only, no-classification round that clears every guard up to
# phase_baseline_is_clean. Callers set the phase verification to vary the baseline.
_seed_clean_guard_round() {
mkdir -p "$PHASE_DIR/remediation/qa/round-01"
printf 'stage=verify\nround=01\n' > "$PHASE_DIR/remediation/qa/.qa-remediation-stage"
create_round_summary_with_files "$PHASE_DIR/remediation/qa/round-01" "01" \
' - "01-test-phase/remediation/qa/round-01/R01-SUMMARY.md"'
cat > "$PHASE_DIR/remediation/qa/round-01/R01-PLAN.md" <<'PLAN'
---
round: 01
title: No-FAIL documentation disposition
---
PLAN
cat > "$PHASE_DIR/remediation/qa/round-01/R01-VERIFICATION.md" <<'VERIF'
---
writer: write-verification.sh
result: PASS
plans_verified:
- R01
---
## Checks
| ID | Category | Description | Status | Evidence |
|----|----------|-------------|--------|----------|
| MH-01 | must_have | Documented | PASS | Done |
VERIF
}

@test "phase_baseline_is_clean branch: no phase verification on disk → fails closed (issue #655)" {
_seed_clean_guard_round
# No phase-level VERIFICATION.md exists (a continuation round that lost its source).

run bash "$SCRIPT" "$PHASE_DIR"

[ "$status" -eq 0 ]
[[ "$output" == *"qa_gate_source_verification_missing=true"* ]]
[[ "$output" == *"qa_gate_routing=REMEDIATION_REQUIRED"* ]]
}

@test "phase_baseline_is_clean branch: phase verification result is not PASS → fails closed (issue #655)" {
create_verif "write-verification.sh" "FAIL"
_seed_clean_guard_round

run bash "$SCRIPT" "$PHASE_DIR"

[ "$status" -eq 0 ]
[[ "$output" == *"qa_gate_source_verification_missing=true"* ]]
[[ "$output" == *"qa_gate_routing=REMEDIATION_REQUIRED"* ]]
}

@test "phase_baseline_is_clean branch: phase verification carries a pre-existing issue → fails closed (issue #655)" {
create_verif "write-verification.sh" "PASS" "## Pre-existing Issues
| Test | File | Error |
|------|------|-------|
| SomeTest | src/foo.swift | flaky boundary case |"
_seed_clean_guard_round

run bash "$SCRIPT" "$PHASE_DIR"

[ "$status" -eq 0 ]
[[ "$output" == *"qa_gate_source_verification_missing=true"* ]]
[[ "$output" == *"qa_gate_routing=REMEDIATION_REQUIRED"* ]]
}

# Issue #655 (round 3, finding 2): a placeholder Pre-existing Issues table (a
# single "None"/"N/A" row) is an explicit empty table, not a carried issue, so
# `phase_baseline_is_clean` must treat the phase as clean — otherwise a clean
# no-FAIL phase could spuriously fail closed.
@test "phase_baseline_is_clean treats a placeholder pre-existing-issues row as clean → PROCEED_TO_UAT (issue #655)" {
create_verif "write-verification.sh" "PASS" "## Pre-existing Issues
| Test | File | Error |
|------|------|-------|
| None | - | - |"
_seed_clean_guard_round

run bash "$SCRIPT" "$PHASE_DIR"

[ "$status" -eq 0 ]
[[ "$output" != *"qa_gate_source_verification_missing=true"* ]]
[[ "$output" == *"qa_gate_routing=PROCEED_TO_UAT"* ]]
}

# Issue #655 (round 3, finding 1): lock in the deliberate broadening for round-02+.
# A round-02 no-FAIL disposition (input_mode=none, zero classifications, metadata-only
# recorded files) on a clean phase baseline terminates rather than looping — the same
# no-FAIL semantics as round-01, exercised through the round>1 source-resolution path.
@test "round-02 no-classification disposition on a clean baseline → PROCEED_TO_UAT (issue #655)" {
create_verif "write-verification.sh" "PASS"
mkdir -p "$PHASE_DIR/remediation/qa/round-01" "$PHASE_DIR/remediation/qa/round-02"
printf 'stage=verify\nround=02\n' > "$PHASE_DIR/remediation/qa/.qa-remediation-stage"
# Prior round-01 verification exists (a normal continuation), PASS/clean.
cat > "$PHASE_DIR/remediation/qa/round-01/R01-VERIFICATION.md" <<'VERIF'
---
writer: write-verification.sh
result: PASS
plans_verified:
- R01
---
## Checks
| ID | Category | Description | Status | Evidence |
|----|----------|-------------|--------|----------|
| MH-01 | must_have | Documented | PASS | Done |
VERIF
create_round_summary_with_files "$PHASE_DIR/remediation/qa/round-02" "02" \
' - "01-test-phase/remediation/qa/round-02/R02-SUMMARY.md"'
cat > "$PHASE_DIR/remediation/qa/round-02/R02-PLAN.md" <<'PLAN'
---
round: 02
title: No-FAIL documentation disposition
---
PLAN
cat > "$PHASE_DIR/remediation/qa/round-02/R02-VERIFICATION.md" <<'VERIF'
---
writer: write-verification.sh
result: PASS
plans_verified:
- R02
---
## Checks
| ID | Category | Description | Status | Evidence |
|----|----------|-------------|--------|----------|
| MH-01 | must_have | Documented | PASS | Done |
VERIF

run bash "$SCRIPT" "$PHASE_DIR"

[ "$status" -eq 0 ]
[[ "$output" != *"qa_gate_source_verification_missing=true"* ]]
[[ "$output" == *"qa_gate_routing=PROCEED_TO_UAT"* ]]
}

@test "metadata-only round with inline fail_classifications process-exception → PROCEED_TO_UAT" {
create_source_fail_verif "DEV-0101-COMMIT" "Historical process exception still needs classification"
create_summary_with_yaml_deviations "01-01" "Historical batched commit"
Expand Down
Loading