melodic-software · kyle-sexton · Aug 23, 2026 · Aug 23, 2026
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -1184,6 +1184,13 @@ jobs:
       - name: Run shared list-file reader tests
         if: needs.scope.outputs.run_full == 'true'
         run: bash scripts/lib/read-list.test.sh
+      # Same reason again — run-plugin-tests.sh never reaches scripts/. This
+      # suite covers scripts/lib/test-harness.sh, the ok/fail/report contract
+      # the scripts/*.test.sh suites share. A recorded failure that still
+      # exits 0 is the false-green this library exists to refuse (#3160).
+      - name: Run shared test-harness tests
+        if: needs.scope.outputs.run_full == 'true'
+        run: bash scripts/lib/test-harness.test.sh
       - name: Validate plugin and catalog manifests
         if: needs.scope.outputs.run_full == 'true'
         run: scripts/validate-plugins.sh

diff --git a/docs/conventions/shell-test-helpers/README.md b/docs/conventions/shell-test-helpers/README.md
@@ -35,9 +35,12 @@ remove, for a mechanism (`check-cross-plugin-source-drift.sh`) that already clas
 outside its scope: they live at different paths per plugin and are not byte-identical, so `discover`
 never flags them as an unregistered cluster.
 
-`scripts/check-skill-portability.test.sh` follows the same reasoning at the repo-tooling layer: it is
-not a plugin, so no plugin assertion library is available to source, and it carries its own minimal
-`PASS`/`FAIL` counters rather than reaching into a plugin's copy.
+Repo-tooling suites under `scripts/` are a different layer: they share
+[`scripts/lib/test-harness.sh`](../../../scripts/lib/test-harness.sh)
+(`ok` / `fail` / `test_harness::report`). That library is not a plugin import
+and does not change the per-plugin rule above.
+`scripts/check-skill-portability.test.sh` sources it rather than reaching into
+a plugin's copy.
 
 ## Exit-code taxonomies also diverge, deliberately
 

diff --git a/scripts/affected-tests.test.sh b/scripts/affected-tests.test.sh
@@ -48,16 +48,8 @@ NO_SUITE="$SELF_DIR/affected-tests-no-suite.txt"
 # shellcheck source=test-git-helpers.sh
 . "$SELF_DIR/test-git-helpers.sh"
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 # A stand-in suite that is cheap to run and records that it ran.
 # shellcheck disable=SC2016 # deliberate: the emitted file must expand these, not this shell
@@ -819,5 +811,4 @@ else
   fail "crossing budget was spent by an incidental hit (rc=$RC): $OUT"
 fi
 
-printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL"
-((FAIL == 0))
+test_harness::report
diff --git a/scripts/check-changed-skills.test.sh b/scripts/check-changed-skills.test.sh
@@ -22,16 +22,8 @@ stage_libs() {
 # shellcheck source=test-git-helpers.sh
 . "$SELF_DIR/test-git-helpers.sh"
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 # A stub checker shared by every scenario: records each invocation (skill name +
 # forwarded env) to $CHECK_LOG, and FAILs iff the skill is named "bad".
@@ -339,6 +331,4 @@ fi
 rm -rf "$r"
 
 rm -f "$STUB"
-echo
-echo "PASS=$PASS FAIL=$FAIL"
-[[ "$FAIL" -eq 0 ]]
+test_harness::report
diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh
@@ -20,16 +20,8 @@ stage_libs() {
 # shellcheck source=test-git-helpers.sh
 . "$SELF_DIR/test-git-helpers.sh"
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 mk_repo() {
   local dir
@@ -1197,5 +1189,4 @@ else
 fi
 rm -rf "$repo"
 
-printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL"
-((FAIL == 0))
+test_harness::report
diff --git a/scripts/check-contract-slice-prune.test.sh b/scripts/check-contract-slice-prune.test.sh
@@ -14,16 +14,8 @@ SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SCRIPT="$SELF_DIR/check-contract-slice-prune.sh"
 PARSE_LIB="$SELF_DIR/../lib/parse-concern-value.sh"
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 # Fixture git commands must not spawn background maintenance. On git >= 2.46,
 # `git commit` and `git merge` fork `git maintenance run --auto --quiet
@@ -471,6 +463,4 @@ repo="$(mk_repo)"
 if (($? == 2)); then ok "unknown mode exits 2"; else fail "unknown mode must exit 2"; fi
 rm -rf "$repo"
 
-echo ""
-echo "check-contract-slice-prune.test.sh: $PASS passed, $FAIL failed"
-((FAIL == 0))
+test_harness::report
diff --git a/scripts/check-cross-plugin-source-drift.test.sh b/scripts/check-cross-plugin-source-drift.test.sh
@@ -11,16 +11,8 @@ set -uo pipefail
 SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SCRIPT="$SELF_DIR/check-cross-plugin-source-drift.sh"
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 # new_fixture → prints the path to a fresh <tmp>/scripts + <tmp>/plugins tree
 # with the script copied in, ready for callers to populate.
@@ -179,6 +171,4 @@ else
   fi
 fi
 
-echo
-echo "PASS=$PASS FAIL=$FAIL"
-[[ "$FAIL" -eq 0 ]]
+test_harness::report
diff --git a/scripts/check-discriminating-test-skips.test.sh b/scripts/check-discriminating-test-skips.test.sh
@@ -12,16 +12,8 @@ REPO_ROOT="$(cd "$SELF_DIR/.." && pwd)"
 SCRIPT="$SELF_DIR/check-discriminating-test-skips.sh"
 HELPERS="$REPO_ROOT/plugins/source-control/scripts/test-helpers.sh"
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 new_fixture() {
   local dir
@@ -169,5 +161,4 @@ else
 fi
 rm -f "$tmp_test"
 
-printf '\n%d passed, %d failed\n' "$PASS" "$FAIL"
-[[ $FAIL -eq 0 ]] || exit 1
+test_harness::report
diff --git a/scripts/check-docs-only.test.sh b/scripts/check-docs-only.test.sh
@@ -23,16 +23,8 @@ ALLOWLIST="$SELF_DIR/docs-only-paths.txt"
 # shellcheck source=test-git-helpers.sh
 . "$SELF_DIR/test-git-helpers.sh"
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 mk_repo() {
   local dir
@@ -150,5 +142,4 @@ else
 fi
 rm -rf "$repo" "$gho"
 
-printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL"
-((FAIL == 0))
+test_harness::report
diff --git a/scripts/check-fleet-audit-doc-grammar.test.sh b/scripts/check-fleet-audit-doc-grammar.test.sh
@@ -7,16 +7,8 @@ set -uo pipefail
 SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SCRIPT="$SELF_DIR/check-fleet-audit-doc-grammar.sh"
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 # Minimal stub collector: classifies the bare-positional probe and the no-scope
 # probe from argv / emptiness without touching git or the network.
@@ -285,5 +277,4 @@ else
   fail "repository skill+parser fails --check"
 fi
 
-echo "PASS=$PASS FAIL=$FAIL"
-[[ "$FAIL" -eq 0 ]]
+test_harness::report
diff --git a/scripts/check-fleet-finding-test-coverage.test.sh b/scripts/check-fleet-finding-test-coverage.test.sh
@@ -7,16 +7,8 @@ set -uo pipefail
 SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SCRIPT="$SELF_DIR/check-fleet-finding-test-coverage.sh"
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 mk_tree() {
   local dir
@@ -220,5 +212,4 @@ else
   fail "repository baseline fails --check"
 fi
 
-echo "PASS=$PASS FAIL=$FAIL"
-[[ "$FAIL" -eq 0 ]]
+test_harness::report
diff --git a/scripts/check-hook-exec-form.test.sh b/scripts/check-hook-exec-form.test.sh
@@ -11,16 +11,8 @@ SCRIPT="$SELF_DIR/check-hook-exec-form.sh"
 READER="$SELF_DIR/check-hook-exec-form-frontmatter.py"
 REQUIREMENTS="$SELF_DIR/../.github/requirements-ci.txt"
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 new_fixture() {
   local dir
@@ -968,6 +960,4 @@ else
 fi
 rm -rf "$f"
 
-echo
-echo "passed: $PASS, failed: $FAIL"
-((FAIL == 0)) || exit 1
+test_harness::report
diff --git a/scripts/check-hook-userconfig-argv.test.sh b/scripts/check-hook-userconfig-argv.test.sh
@@ -18,16 +18,8 @@ stage_libs() {
   cp "$SELF_DIR/lib/read-list.sh" "$1/lib/"
 }
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 new_fixture() {
   local dir
@@ -293,6 +285,4 @@ else
 fi
 rm -rf "$f"
 
-echo
-echo "passed: $PASS, failed: $FAIL"
-((FAIL == 0)) || exit 1
+test_harness::report
diff --git a/scripts/check-orphaned-fixtures.test.sh b/scripts/check-orphaned-fixtures.test.sh
@@ -17,16 +17,8 @@ stage_libs() {
   cp "$SELF_DIR/lib/read-list.sh" "$1/lib/"
 }
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 # mk_repo <baseline-content>: throwaway repo with the detector installed and a
 # baseline file at scripts/orphaned-fixtures-baseline.txt.
@@ -185,5 +177,4 @@ repo="$(mk_repo)"
 if [[ $? -eq 2 ]]; then ok "bad mode -> exit 2"; else fail "bad mode did not exit 2"; fi
 rm -rf "$repo"
 
-printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL"
-((FAIL == 0))
+test_harness::report
diff --git a/scripts/check-queue-front-matter.test.sh b/scripts/check-queue-front-matter.test.sh
@@ -5,16 +5,8 @@ set -uo pipefail
 SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 SCRIPT="$SELF_DIR/check-queue-front-matter.sh"
 
-PASS=0
-FAIL=0
-fail() {
-  echo "FAIL: $*" >&2
-  FAIL=$((FAIL + 1))
-}
-ok() {
-  echo "ok: $*"
-  PASS=$((PASS + 1))
-}
+# shellcheck source=lib/test-harness.sh
+. "$SELF_DIR/lib/test-harness.sh"
 
 new_queue() {
   mktemp -d
@@ -103,5 +95,4 @@ else
   fail "README should be ignored: valid sole item should pass"
 fi
 
-printf '\n%d passed, %d failed\n' "$PASS" "$FAIL"
-[[ "$FAIL" -eq 0 ]]
+test_harness::report