diff --git a/docs/governance-campaign-findings.md b/docs/governance-campaign-findings.md index 0cc6addc..b3addfa7 100644 --- a/docs/governance-campaign-findings.md +++ b/docs/governance-campaign-findings.md @@ -390,3 +390,38 @@ assumption that the build path was the only route to a file sink. The only defence that worked was **running the degraded case** — reverting the fix and confirming the assertion fails. An assertion never observed failing has not been tested, it has been written. + +A systematic audit followed, because four accidental discoveries are a poor +sampling method. Scope: 360 assertion sites in `run.sh` and 2294 across 235 +shell suites. The result was mostly reassuring — most absence checks already +pair with a positive existence check, a fail-safe default (`${X:-1}`, where +unset means *fail*), or a named control (`TR-01` is labelled "the +anti-regression"; `PU-01` fails outright if its scenario produced fewer than six +samples). What it did find: + +| finding | shape | +|---|---| +| `H01` | health verdict inferred from stderr containing the string `"governance"` — which every governed run prints. Passed hardest on a run that died before reporting its own health. | +| 25 sites / 7 security suites | the suite defined only `ok()` and `fail()` — **no `skip()`** — so "executor not available" had nowhere to go but PASS | +| `L24-03` | trivially true at zero attestations; non-vacuous only because a *neighbouring* assertion stayed strict | +| `DB-01`, `DB-03`, `DB-04` | initialiser `0` and fallback `"1.00 0 0"` are indistinguishable from "the scenario never ran" | +| `E04` | asserted a line was absent from a dashboard that a failed run never produced | + +`test_gov_comment_styles_gov002.sh` is the one to remember: it reported +**"Results: 4/4 passed"** while verifying nothing at all — every check +unverifiable in that environment. It now reports `0/4 passed, 4 skipped`. + +The generalisable rule is narrower than "write better assertions": **an +assertion about an absence needs the thing it is absent from to exist.** A +missing line in a document nobody wrote, a zero from a counter nobody +incremented, and a clean result are the same observation until something +distinguishes them. + +One postscript, because it is the same lesson at one remove. The first pass +found 19 of those 25 sites; reviewing the diff before merge found six more in +files already being edited. The filter had searched for `not available` and +`may be`, and the survivors read *"may **not be** available"*, *"depth may not +have been reached"*, and *"(acceptable)"*. A keyword filter over prose is itself +an assertion that can be satisfied without the property holding — it finds the +phrasings you thought of. Reading the changed files end to end is what caught +the rest. diff --git a/examples/living-script_extended/run.sh b/examples/living-script_extended/run.sh index 1a6a0e48..605cda93 100644 --- a/examples/living-script_extended/run.sh +++ b/examples/living-script_extended/run.sh @@ -1478,7 +1478,14 @@ assert 'refinement_iterations' in d EV_SIGNED=$(echo "$OUTPUT" | grep -oP 'EVIDENCE\|signed=\K[0-9]+' | head -1) EV_FP=$(echo "$OUTPUT" | grep -oP 'EVIDENCE\|signed=[0-9]+\|fingerprinted=\K[0-9]+' | head -1) EV_EMPTY_FP=$(echo "$OUTPUT" | grep -oP 'EVIDENCE\|.*\|empty_fingerprint=\K[0-9]+' | head -1) - if [ "${EV_SIGNED:-0}" -eq "${EV_ATT:-(-1)}" ] && [ "${EV_FP:-0}" -eq "${EV_ATT:-(-1)}" ] \ + # Self-guarding on purpose. With zero attestations every equality below holds + # trivially and this reported "0 signed, 0 empty fingerprints" as a pass. It + # was non-vacuous only because L24-02 separately fails when nothing attested + # — correct today, but a claim that depends on a neighbour staying strict is + # one edit away from silently proving nothing. Require the subject to exist. + if [ "${EV_ATT:-0}" -lt 1 ]; then + skip "L24-03" "No attestations to check — signing is unverified this run (L24-02 owns the floor)" + elif [ "${EV_SIGNED:-0}" -eq "${EV_ATT:-(-1)}" ] && [ "${EV_FP:-0}" -eq "${EV_ATT:-(-1)}" ] \ && [ "${EV_EMPTY_FP:-1}" -eq 0 ]; then pass "L24-03" "Every attestation is signed and attributable (${EV_SIGNED} signed, 0 empty fingerprints)" else @@ -1820,15 +1827,28 @@ print('true' if ok else 'false') fi [ "${TELEM_CHAIN:-}" = "true" ] && pass "T01" "Telemetry hash chain valid" || skip "T01" "Hash chain not verified" + # The script prints "HEALTH: " from governance.health() as its very + # last act, so that line is present exactly when the run reached the end. + # + # The stderr fallback used to invent a verdict when the line was missing: + # "IMPAIRED" anywhere in stderr meant impaired, and then the mere presence of + # the string "governance" meant healthy. Every governed run prints that word, + # so this gk_fail-severity check passed on a substring match — and it passed + # hardest in the case that matters most, a run that died before it could + # report its own health. IMPAIRED in stderr is still real evidence and still + # fails; absence is now inconclusive rather than healthy. HEALTH=$(echo "$OUTPUT" | grep -oP 'HEALTH: \K\w+' | head -1) - if [ -z "$HEALTH" ] && [ -f "$STDERR_FILE" ]; then - grep -q "IMPAIRED" "$STDERR_FILE" 2>/dev/null && HEALTH="impaired" - grep -q "governance" "$STDERR_FILE" 2>/dev/null && HEALTH="${HEALTH:-healthy}" + HEALTH_SRC="script" + if [ -z "$HEALTH" ] && [ -f "$STDERR_FILE" ] \ + && grep -q "IMPAIRED" "$STDERR_FILE" 2>/dev/null; then + HEALTH="impaired"; HEALTH_SRC="stderr" fi if [ "${HEALTH:-}" = "healthy" ] || [ "${HEALTH:-}" = "degraded" ]; then pass "H01" "Governance health: $HEALTH" + elif [ -z "${HEALTH:-}" ]; then + skip "H01" "No health verdict emitted — the run never reached governance.health() (see R01)" else - gk_fail "H01" "Governance health: ${HEALTH:-unknown}" + gk_fail "H01" "Governance health: ${HEALTH}" "reported via ${HEALTH_SRC}" fi # Code extraction diff --git a/tests/governance_v4/test_developer_blindspot.sh b/tests/governance_v4/test_developer_blindspot.sh index 577eb064..65427fcf 100755 --- a/tests/governance_v4/test_developer_blindspot.sh +++ b/tests/governance_v4/test_developer_blindspot.sh @@ -274,7 +274,11 @@ printf " %-14s %-26s %-26s %s\n" "" "floor/quar/fired" "floor/quar/fired" "" echo " --------------------------------------------------------------------------------------" BLIND="" -FULL_FP=0; DEV_FP=0 +# Sentinel, not 0. DB-01 asserts the compliant control was NOT quarantined, and +# an initialiser of 0 is indistinguishable from "the compliant scenario never +# ran" — the assertion would confirm no false positives on a measurement that +# never happened. -1 means unmeasured and makes DB-01 skip instead. +FULL_FP=-1; DEV_FP=-1 DEV_MOVED=0 for b in "${BEHAVIOURS[@]}"; do read -r F_FLOOR F_QUAR F_FIRED <<< "$(measure "f-$b" "$b" full)" @@ -310,7 +314,9 @@ echo "" # DB-01 — the control. An override set that quarantines good code is worse than # the blind spot it was added to prevent. -if [ "${DEV_FP:-0}" -eq 0 ]; then +if [ "${DEV_FP:--1}" -lt 0 ]; then + skip "DB-01" "compliant control never measured — no-false-positive claim unverified" +elif [ "${DEV_FP:-0}" -eq 0 ]; then pass "DB-01" "developer overrides do not quarantine compliant code" else fail "DB-01" "developer overrides quarantined compliant code $DEV_FP times" \ @@ -329,7 +335,9 @@ fi # DB-03 — the blind-spot ledger. Recorded rather than assumed: this is the # number that says whether tuning the overrides cost real coverage. -if [ -z "$BLIND" ]; then +if [ "${DEV_FP:--1}" -lt 0 ]; then + skip "DB-03" "no behaviour was measured — an empty blind-spot ledger means nothing" +elif [ -z "$BLIND" ]; then pass "DB-03" "overrides cost no detection coverage on the tested behaviours" else fail "DB-03" "overrides lose coverage:$BLIND" \ @@ -340,13 +348,21 @@ fi # failure mode and no CDD signal sees it. Asserting the absence keeps it in the # report rather than leaving a green run to imply coverage that does not exist; # if a future signal catches it, this fails and gets deleted, which is the point. -read -r E_FLOOR E_QUAR E_FIRED <<< "$(cat "$TEST_TMP/.erosion.res" 2>/dev/null || echo "1.00 0 0")" +# The fallback used to be "1.00 0 0", which reads as a confirmed known-negative: +# a missing result file produced the same quar=0/fired=0 as an erosion scenario +# that ran and was genuinely invisible. Assert the known negative only when the +# scenario actually produced a measurement. +if [ ! -f "$TEST_TMP/.erosion.res" ]; then + skip "DB-04" "erosion scenario produced no measurement — known negative unconfirmed this run" +else +read -r E_FLOOR E_QUAR E_FIRED <<< "$(cat "$TEST_TMP/.erosion.res")" if [ "${E_QUAR:-0}" -eq 0 ] && [ "${E_FIRED:-0}" -eq 0 ]; then pass "DB-04" "erosion invisible to CDD from responses alone, no ground truth fed (known negative, floor=$E_FLOOR)" else fail "DB-04" "erosion now scores (quar=$E_QUAR fired=$E_FIRED) — the known negative is stale" \ "a signal has started catching this; update the finding rather than the threshold" fi +fi echo "" echo "────────────────────────────────────────" diff --git a/tests/governance_v4/test_escalation_effectiveness.sh b/tests/governance_v4/test_escalation_effectiveness.sh index c137bc1f..53924ccf 100644 --- a/tests/governance_v4/test_escalation_effectiveness.sh +++ b/tests/governance_v4/test_escalation_effectiveness.sh @@ -184,7 +184,13 @@ export NAAB_TEST_FAKE_KEY="fake-key-for-create-only" OUT=$("$NAAB" --governance-dashboard "$WORK/test.naab" 2>&1) || true ESC_LINE=$(echo "$OUT" | grep "Escalation:" | wc -l) ESC_LINE=$(echo "$ESC_LINE" | tr -d ' ') -if [ "$ESC_LINE" -eq 0 ]; then +# The dashboard has to exist before its contents can be asserted about. A run +# that failed outright emits no dashboard, zero "Escalation:" lines, and used to +# read as "correctly quiet" — the absence of a line in a document that was never +# written. Require the document. +if ! echo "$OUT" | grep -q "Agent Governance Summary"; then + skip "E04" "no dashboard produced — an absent 'Escalation:' line proves nothing" +elif [ "$ESC_LINE" -eq 0 ]; then pass "E04" "No 'Escalation:' line when no escalation occurred" else fail "E04" "Unexpected 'Escalation:' line in dashboard" "found $ESC_LINE lines" diff --git a/tests/security/test_gov_comment_styles_gov002.sh b/tests/security/test_gov_comment_styles_gov002.sh index b3d4c230..d05d6a28 100755 --- a/tests/security/test_gov_comment_styles_gov002.sh +++ b/tests/security/test_gov_comment_styles_gov002.sh @@ -8,10 +8,14 @@ set -euo pipefail NAAB="${1:-$(dirname "$0")/../../build/naab-lang}" -PASS=0; FAIL=0 +PASS=0; FAIL=0; SKIP=0 ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } +# A run where the executor is absent has not verified anything. Without this +# the suite could only say PASS or FAIL, so "SQL executor not available" was +# recorded as a pass — a green result standing in for an unrun check. +skip() { echo " SKIP: $1"; SKIP=$((SKIP + 1)); } WORK_DIR="${HOME}/.naab/gov002_$$" mkdir -p "$WORK_DIR" @@ -59,7 +63,7 @@ NAAB ec=0 out=$("$NAAB" "$WORK_DIR/t1.naab" --no-governance 2>&1) || ec=$? if [[ "$ec" -ne 0 ]]; then - ok "T1 skipped — SQL executor not available: ${out:0:80}" + skip "T1 skipped — SQL executor not available: ${out:0:80}" else ec2=0 out2=$("$NAAB" "$WORK_DIR/t1.naab" 2>&1) || ec2=$? @@ -68,7 +72,7 @@ else elif echo "$out2" | grep -qi "FORBIDDEN\|hallucinated\|custom.*rule\|governance\|blocked"; then fail "False positive: SQL -- comment triggered custom pattern: ${out2:0:120}" else - ok "SQL executor not available or other non-governance exit: ${out2:0:80}" + skip "SQL executor not available or other non-governance exit: ${out2:0:80}" fi fi @@ -91,16 +95,16 @@ NAAB ec=0 out=$("$NAAB" "$WORK_DIR/t2.naab" --no-governance 2>&1) || ec=$? if [[ "$ec" -ne 0 ]]; then - ok "T2 skipped — SQL executor not available: ${out:0:80}" + skip "T2 skipped — SQL executor not available: ${out:0:80}" else ec2=0 out2=$("$NAAB" "$WORK_DIR/t2.naab" 2>&1) || ec2=$? if echo "$out2" | grep -qi "FORBIDDEN\|hallucinated\|custom.*rule\|governance\|blocked\|denied"; then ok "Real FORBIDDEN_KEYWORD in SQL correctly blocked — custom pattern active" elif [[ "$ec2" -ne 0 ]]; then - ok "SQL block with forbidden keyword produced non-zero exit (exit $ec2)" + skip "SQL block with forbidden keyword produced non-zero exit (exit $ec2)" else - ok "SQL executor ran without block (governance may not apply to SQL blocks without executor)" + skip "SQL executor ran without block (governance may not apply to SQL blocks without executor)" fi fi @@ -131,7 +135,7 @@ NAAB ec=0 out=$("$NAAB" "$WORK_DIR/t3.naab" --no-governance 2>&1) || ec=$? if [[ "$ec" -ne 0 ]]; then - ok "T3 skipped — SQL executor not available" + skip "T3 skipped — SQL executor not available" else ec2=0 out2=$("$NAAB" "$WORK_DIR/t3.naab" 2>&1) || ec2=$? @@ -140,7 +144,7 @@ else elif echo "$out2" | grep -qi "temporary\|for now\|governance\|blocked"; then fail "SQL -- comment unexpectedly triggered governance: ${out2:0:120}" else - ok "SQL executor not available or other non-governance exit (exit $ec2)" + skip "SQL executor not available or other non-governance exit (exit $ec2)" fi fi @@ -165,7 +169,7 @@ NAAB ec=0 out=$("$NAAB" "$WORK_DIR/t4.naab" --no-governance 2>&1) || ec=$? if [[ "$ec" -ne 0 ]] || ! echo "$out" | grep -q "clean"; then - ok "T4 skipped — Python not available: ${out:0:80}" + skip "T4 skipped — Python not available: ${out:0:80}" else ec2=0 out2=$("$NAAB" "$WORK_DIR/t4.naab" 2>&1) || ec2=$? @@ -174,11 +178,11 @@ else elif echo "$out2" | grep -qi "FORBIDDEN\|hallucinated\|governance\|blocked"; then fail "False positive: Python # comment triggered custom pattern: ${out2:0:120}" else - ok "Completed (Python may be unavailable): ${out2:0:80}" + skip "Completed (Python may be unavailable): ${out2:0:80}" fi fi echo "" -TOTAL=$(( PASS + FAIL )) -echo "Results: ${PASS}/${TOTAL} passed" +TOTAL=$(( PASS + FAIL + SKIP )) +echo "Results: ${PASS}/${TOTAL} passed, ${SKIP} skipped (unverified)" if [[ "$FAIL" -gt 0 ]]; then exit 1; fi diff --git a/tests/security/test_gov_string_prefix_gov001.sh b/tests/security/test_gov_string_prefix_gov001.sh index 04f1469f..26996912 100755 --- a/tests/security/test_gov_string_prefix_gov001.sh +++ b/tests/security/test_gov_string_prefix_gov001.sh @@ -6,10 +6,14 @@ set -euo pipefail NAAB="${1:-$(dirname "$0")/../../build/naab-lang}" -PASS=0; FAIL=0 +PASS=0; FAIL=0; SKIP=0 ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } +# A run whose executor is absent, or whose failure cannot be attributed, has +# verified nothing. Without this the suite could only say PASS or FAIL, so +# "not available" was recorded as a pass — a green standing in for an unrun check. +skip() { echo " SKIP: $1"; SKIP=$((SKIP + 1)); } WORK_DIR="${HOME}/.naab/gov001_$$" mkdir -p "$WORK_DIR/injection" "$WORK_DIR/placeholder" @@ -58,7 +62,7 @@ NAAB ec=0 out=$("$NAAB" "$WORK_DIR/injection/t1.naab" --no-governance 2>&1) || ec=$? if [[ "$ec" -ne 0 ]] || ! echo "$out" | grep -q "ok"; then - ok "T1 skipped — Python not available or unexpected output: ${out:0:80}" + skip "T1 skipped — Python not available or unexpected output: ${out:0:80}" else ec2=0 out2=$("$NAAB" "$WORK_DIR/injection/t1.naab" 2>&1) || ec2=$? @@ -67,7 +71,7 @@ else elif echo "$out2" | grep -qi "injection\|blocked\|governance\|command"; then fail "False positive: f-string content triggered command-injection: ${out2:0:120}" else - ok "Script completed (Python may be unavailable or governance not triggered): ${out2:0:80}" + skip "Script completed (Python may be unavailable or governance not triggered): ${out2:0:80}" fi fi @@ -93,7 +97,7 @@ NAAB ec=0 out=$("$NAAB" "$WORK_DIR/injection/t2.naab" --no-governance 2>&1) || ec=$? if [[ "$ec" -ne 0 ]] || ! echo "$out" | grep -q "safe"; then - ok "T2 skipped — Python not available: ${out:0:80}" + skip "T2 skipped — Python not available: ${out:0:80}" else ec2=0 out2=$("$NAAB" "$WORK_DIR/injection/t2.naab" 2>&1) || ec2=$? @@ -102,7 +106,7 @@ else elif echo "$out2" | grep -qi "injection\|blocked\|governance"; then fail "False positive: r-string content triggered injection check: ${out2:0:120}" else - ok "Completed (Python may be unavailable): ${out2:0:80}" + skip "Completed (Python may be unavailable): ${out2:0:80}" fi fi @@ -128,7 +132,7 @@ NAAB ec=0 out=$("$NAAB" "$WORK_DIR/injection/t3.naab" --no-governance 2>&1) || ec=$? if [[ "$ec" -ne 0 ]] || ! echo "$out" | grep -qi "ran\|dangerous"; then - ok "T3 skipped — Python not available: ${out:0:80}" + skip "T3 skipped — Python not available: ${out:0:80}" else ec2=0 out2=$("$NAAB" "$WORK_DIR/injection/t3.naab" 2>&1) || ec2=$? @@ -184,6 +188,6 @@ else fi echo "" -TOTAL=$(( PASS + FAIL )) -echo "Results: ${PASS}/${TOTAL} passed" +TOTAL=$(( PASS + FAIL + SKIP )) +echo "Results: ${PASS}/${TOTAL} passed, ${SKIP} skipped (unverified)" if [[ "$FAIL" -gt 0 ]]; then exit 1; fi diff --git a/tests/security/test_js_marshal_depth_rt006.sh b/tests/security/test_js_marshal_depth_rt006.sh index 48194460..3c3e83c0 100755 --- a/tests/security/test_js_marshal_depth_rt006.sh +++ b/tests/security/test_js_marshal_depth_rt006.sh @@ -8,10 +8,14 @@ set -euo pipefail NAAB="${1:-$(dirname "$0")/../../build/naab-lang}" -PASS=0; FAIL=0 +PASS=0; FAIL=0; SKIP=0 ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } +# A run whose executor is absent, or whose failure cannot be attributed, has +# verified nothing. Without this the suite could only say PASS or FAIL, so +# "not available" was recorded as a pass — a green standing in for an unrun check. +skip() { echo " SKIP: $1"; SKIP=$((SKIP + 1)); } WORK_DIR="${HOME}/.naab/rt006_$$" mkdir -p "$WORK_DIR" @@ -57,7 +61,7 @@ elif [[ "$ec" -ne 0 ]]; then ok "Non-zero exit without crash — depth limit or JS exception caught (exit $ec)" else # If JS executor is not available, this might just succeed with truncated data - ok "Completed without crash — JS may not be available or depth was handled" + skip "Completed without crash — JS may not be available or depth was handled" fi echo " Output: ${out:0:120}" @@ -96,14 +100,14 @@ elif [[ "$ec" -eq 0 ]] && echo "$out" | grep -q "ok"; then elif [[ "$ec" -ne 0 ]]; then # JS executor might not be available if echo "$out" | grep -qi "javascript.*not.*available\|executor.*not\|not.*compiled\|not.*built"; then - ok "T2 skipped — JavaScript executor not available" + skip "T2 skipped — JavaScript executor not available" elif echo "$out" | grep -qi "depth\|maximum\|RangeError"; then fail "Depth error at only 30 levels — false positive depth guard" else - ok "Non-zero exit (exit $ec) — JS may not be available: ${out:0:80}" + skip "Non-zero exit (exit $ec) — JS may not be available: ${out:0:80}" fi else - ok "T2 completed (may not have printed 'ok' if JS unavailable): ${out:0:80}" + skip "T2 completed (may not have printed 'ok' if JS unavailable): ${out:0:80}" fi echo " Output: ${out:0:120}" @@ -140,14 +144,14 @@ elif [[ "$ec" -eq 124 ]]; then elif echo "$out" | grep -qi "depth\|maximum\|marshall\|exceeded\|RangeError"; then ok "Clear depth error for nested dict — no crash" elif [[ "$ec" -ne 0 ]]; then - ok "Non-zero exit without crash — depth limit caught or JS unavailable (exit $ec)" + skip "Non-zero exit without crash — depth limit caught or JS unavailable (exit $ec)" else - ok "Completed without crash (JS may not be available or dict handled)" + skip "Completed without crash (JS may not be available or dict handled)" fi echo " Output: ${out:0:120}" echo "" -TOTAL=$(( PASS + FAIL )) -echo "Results: ${PASS}/${TOTAL} passed" +TOTAL=$(( PASS + FAIL + SKIP )) +echo "Results: ${PASS}/${TOTAL} passed, ${SKIP} skipped (unverified)" if [[ "$FAIL" -gt 0 ]]; then exit 1; fi diff --git a/tests/security/test_marshal_depth_rt005.sh b/tests/security/test_marshal_depth_rt005.sh index e2090d40..db8806b0 100755 --- a/tests/security/test_marshal_depth_rt005.sh +++ b/tests/security/test_marshal_depth_rt005.sh @@ -5,10 +5,14 @@ set -euo pipefail NAAB="${1:-$(dirname "$0")/../../build/naab-lang}" -PASS=0; FAIL=0 +PASS=0; FAIL=0; SKIP=0 ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } +# A run whose executor is absent, or whose failure cannot be attributed, has +# verified nothing. Without this the suite could only say PASS or FAIL, so +# "not available" was recorded as a pass — a green standing in for an unrun check. +skip() { echo " SKIP: $1"; SKIP=$((SKIP + 1)); } WORKDIR="${HOME}/.naab/test_rt005_$$" mkdir -p "$WORKDIR" @@ -72,7 +76,7 @@ if [[ "$ec" -eq 139 ]] || [[ "$ec" -eq 134 ]]; then elif echo "$out" | grep -qi "depth\|maximum.*depth\|marshalling error\|nested.*structure"; then ok "clear depth error from valueToPyObject — no crash" elif [[ "$ec" -ne 0 ]]; then - ok "non-zero exit without crash — depth limit or other error caught" + skip "non-zero exit without crash — depth limit or other error caught" else ok "completed without crash (depth guard active or executor handled gracefully)" fi @@ -110,13 +114,13 @@ elif [[ "$ec" -eq 0 ]]; then ok "30-level list marshalled without error" else if echo "$out" | grep -qi "executor\|python\|not found\|not available"; then - ok "no Python executor — depth limit not triggered (acceptable)" + skip "no Python executor — depth limit not triggered (acceptable)" else ok "non-zero exit for non-depth reason: ${out:0:80}" fi fi echo "" -TOTAL=$(( PASS + FAIL )) -echo "Results: ${PASS}/${TOTAL} passed" +TOTAL=$(( PASS + FAIL + SKIP )) +echo "Results: ${PASS}/${TOTAL} passed, ${SKIP} skipped (unverified)" if [[ "$FAIL" -gt 0 ]]; then exit 1; fi diff --git a/tests/security/test_polyglot_exception_taint_d.sh b/tests/security/test_polyglot_exception_taint_d.sh index 2070c11f..95ffe3fe 100755 --- a/tests/security/test_polyglot_exception_taint_d.sh +++ b/tests/security/test_polyglot_exception_taint_d.sh @@ -5,9 +5,13 @@ set -euo pipefail NAAB="${1:-$(dirname "$0")/../../build/naab-lang}" -PASS=0; FAIL=0 +PASS=0; FAIL=0; SKIP=0 ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } +# A run whose executor is absent, or whose failure cannot be attributed, has +# verified nothing. Without this the suite could only say PASS or FAIL, so +# "not available" was recorded as a pass — a green standing in for an unrun check. +skip() { echo " SKIP: $1"; SKIP=$((SKIP + 1)); } GOVDIR="${HOME}/.naab/test_govD_$$" mkdir -p "$GOVDIR" @@ -160,7 +164,7 @@ elif [[ "$ec" -eq 0 ]]; then else # Non-zero is OK if it's because shell executor isn't available if echo "$out" | grep -qi "executor\|not.*available\|python\|shell"; then - ok "executor not available — taint scoping not testable (acceptable)" + skip "executor not available — taint scoping not testable (acceptable)" else fail "unexpected failure: exit $ec: ${out:0:120}" fi @@ -168,6 +172,6 @@ fi echo "" -TOTAL=$((PASS + FAIL)) -echo "Results: ${PASS}/${TOTAL} passed" +TOTAL=$((PASS + FAIL + SKIP)) +echo "Results: ${PASS}/${TOTAL} passed, ${SKIP} skipped (unverified)" [[ "$FAIL" -eq 0 ]] diff --git a/tests/security/test_r24_fixes.sh b/tests/security/test_r24_fixes.sh index ef6f73f3..0cc6004a 100644 --- a/tests/security/test_r24_fixes.sh +++ b/tests/security/test_r24_fixes.sh @@ -24,9 +24,13 @@ trap cleanup EXIT PASS=0 FAIL=0 +SKIP=0 pass() { echo "PASS: $1"; PASS=$((PASS+1)); } fail() { echo "FAIL: $1"; [[ -n "${2:-}" ]] && echo " $2"; FAIL=$((FAIL+1)); } +# "introspection unavailable" verifies nothing; recording it as a pass let an +# unrun check read as a green one. +skip() { echo "SKIP: $1"; SKIP=$((SKIP+1)); } # ── V-API-004: REST API per-request timeout ───────────────────────────────── @@ -153,7 +157,7 @@ if strings "$NAAB" 2>/dev/null | grep -q 'MAXFILESIZE\|module_resolver' || \ pass "V-DOS-003-T1: naab-lang binary links libcurl (size cap is compile-time)" else # Not all distros bundle strings; this isn't a hard failure - pass "V-DOS-003-T1: introspection unavailable — skipped" + skip "V-DOS-003-T1: introspection unavailable — link check unverified" fi # Test 6: Backslash sanitization is internal to urlToCachePath — no public @@ -260,5 +264,5 @@ fi # ── Summary ───────────────────────────────────────────────────────────────── echo "" -echo "Results: $PASS passed, $FAIL failed" +echo "Results: $PASS passed, $FAIL failed, $SKIP skipped (unverified)" [[ $FAIL -eq 0 ]] diff --git a/tests/security/test_serialize_depth_vm002.sh b/tests/security/test_serialize_depth_vm002.sh index 22156064..f5390b54 100755 --- a/tests/security/test_serialize_depth_vm002.sh +++ b/tests/security/test_serialize_depth_vm002.sh @@ -4,9 +4,13 @@ set -euo pipefail NAAB="${1:-$(dirname "$0")/../../build/naab-lang}" -PASS=0; FAIL=0 +PASS=0; FAIL=0; SKIP=0 ok() { echo " PASS: $1"; PASS=$((PASS + 1)); } fail() { echo " FAIL: $1"; FAIL=$((FAIL + 1)); } +# A run whose executor is absent, or whose failure cannot be attributed, has +# verified nothing. Without this the suite could only say PASS or FAIL, so +# "not available" was recorded as a pass — a green standing in for an unrun check. +skip() { echo " SKIP: $1"; SKIP=$((SKIP + 1)); } WORKDIR="${HOME}/.naab/test_vm002_$$" mkdir -p "$WORKDIR" @@ -48,10 +52,10 @@ if [[ "$ec" -eq 139 ]] || [[ "$ec" -eq 134 ]]; then elif echo "$out" | grep -qi "depth\|serialization error\|maximum"; then ok "clear depth error produced — no crash" elif [[ "$ec" -ne 0 ]]; then - ok "non-zero exit without crash — depth limit or other error caught" + skip "non-zero exit without crash — depth limit or other error caught" else # Might succeed if the dict is not deep enough to trigger on this platform - ok "completed without crash (depth may not have been reached)" + skip "completed without crash (depth may not have been reached)" fi echo "" @@ -88,7 +92,7 @@ elif [[ "$ec" -eq 0 ]]; then else # Non-zero exit for other reasons (no Python executor, etc.) is fine if echo "$out" | grep -qi "executor\|python\|not found\|not available"; then - ok "no Python executor — depth limit not triggered (acceptable)" + skip "no Python executor — depth limit not triggered (acceptable)" else ok "non-zero exit for non-depth reason: ${out:0:80}" fi @@ -96,6 +100,6 @@ fi echo "" -TOTAL=$((PASS + FAIL)) -echo "Results: ${PASS}/${TOTAL} passed" +TOTAL=$((PASS + FAIL + SKIP)) +echo "Results: ${PASS}/${TOTAL} passed, ${SKIP} skipped (unverified)" [[ "$FAIL" -eq 0 ]]