From d90836c27f9cb901f724f8a4e14dfefb6a0681b7 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 23 Aug 2026 22:50:34 +0000 Subject: [PATCH] refactor(scripts): unify test-harness scaffolding in scripts/lib/test-harness.sh Add a sourced-only library that owns ok/fail/report and the exit contract, then migrate the 23 PASS/FAIL-dialect suites onto it. TAP-ish suites stay on their own dialect this pass. The harness suite asserts a recorded failure cannot exit 0 when report is the last command. Closes #3160 Co-authored-by: Kyle Sexton --- .github/workflows/ci.yml | 7 ++ docs/conventions/shell-test-helpers/README.md | 9 +- scripts/affected-tests.test.sh | 15 +-- scripts/check-changed-skills.test.sh | 16 +-- scripts/check-changelog-parity.test.sh | 15 +-- scripts/check-contract-slice-prune.test.sh | 16 +-- .../check-cross-plugin-source-drift.test.sh | 16 +-- .../check-discriminating-test-skips.test.sh | 15 +-- scripts/check-docs-only.test.sh | 15 +-- scripts/check-fleet-audit-doc-grammar.test.sh | 15 +-- .../check-fleet-finding-test-coverage.test.sh | 15 +-- scripts/check-hook-exec-form.test.sh | 16 +-- scripts/check-hook-userconfig-argv.test.sh | 16 +-- scripts/check-orphaned-fixtures.test.sh | 15 +-- scripts/check-queue-front-matter.test.sh | 15 +-- scripts/check-shell-portability.test.sh | 21 +--- scripts/check-silent-revert.test.sh | 16 +-- scripts/check-silent-skips.test.sh | 16 +-- scripts/check-skill-portability.test.sh | 21 +--- .../check-skill-precompute-compose.test.sh | 15 +-- scripts/generate-cheatsheet.test.sh | 16 +-- scripts/lib/changed-files.test.sh | 18 +-- scripts/lib/read-list.test.sh | 16 +-- scripts/lib/test-harness.sh | 49 ++++++++ scripts/lib/test-harness.test.sh | 118 ++++++++++++++++++ scripts/sync-standards-contract.test.sh | 16 +-- scripts/test-git-helpers.test.sh | 16 +-- 27 files changed, 253 insertions(+), 301 deletions(-) create mode 100644 scripts/lib/test-harness.sh create mode 100755 scripts/lib/test-harness.test.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7402c954c1..95db169191 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1184,6 +1184,13 @@ jobs: - name: Run shared list-file reader tests if: needs.scope.outputs.run_full == 'true' run: bash scripts/lib/read-list.test.sh + # Same reason again — run-plugin-tests.sh never reaches scripts/. This + # suite covers scripts/lib/test-harness.sh, the ok/fail/report contract + # the scripts/*.test.sh suites share. A recorded failure that still + # exits 0 is the false-green this library exists to refuse (#3160). + - name: Run shared test-harness tests + if: needs.scope.outputs.run_full == 'true' + run: bash scripts/lib/test-harness.test.sh - name: Validate plugin and catalog manifests if: needs.scope.outputs.run_full == 'true' run: scripts/validate-plugins.sh diff --git a/docs/conventions/shell-test-helpers/README.md b/docs/conventions/shell-test-helpers/README.md index 1feb5c2181..fd8163a601 100644 --- a/docs/conventions/shell-test-helpers/README.md +++ b/docs/conventions/shell-test-helpers/README.md @@ -35,9 +35,12 @@ remove, for a mechanism (`check-cross-plugin-source-drift.sh`) that already clas outside its scope: they live at different paths per plugin and are not byte-identical, so `discover` never flags them as an unregistered cluster. -`scripts/check-skill-portability.test.sh` follows the same reasoning at the repo-tooling layer: it is -not a plugin, so no plugin assertion library is available to source, and it carries its own minimal -`PASS`/`FAIL` counters rather than reaching into a plugin's copy. +Repo-tooling suites under `scripts/` are a different layer: they share +[`scripts/lib/test-harness.sh`](../../../scripts/lib/test-harness.sh) +(`ok` / `fail` / `test_harness::report`). That library is not a plugin import +and does not change the per-plugin rule above. +`scripts/check-skill-portability.test.sh` sources it rather than reaching into +a plugin's copy. ## Exit-code taxonomies also diverge, deliberately diff --git a/scripts/affected-tests.test.sh b/scripts/affected-tests.test.sh index 20c5389ab1..b5c0072a29 100755 --- a/scripts/affected-tests.test.sh +++ b/scripts/affected-tests.test.sh @@ -48,16 +48,8 @@ NO_SUITE="$SELF_DIR/affected-tests-no-suite.txt" # shellcheck source=test-git-helpers.sh . "$SELF_DIR/test-git-helpers.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" # A stand-in suite that is cheap to run and records that it ran. # shellcheck disable=SC2016 # deliberate: the emitted file must expand these, not this shell @@ -819,5 +811,4 @@ else fail "crossing budget was spent by an incidental hit (rc=$RC): $OUT" fi -printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL" -((FAIL == 0)) +test_harness::report diff --git a/scripts/check-changed-skills.test.sh b/scripts/check-changed-skills.test.sh index 63991403eb..38f06034bf 100755 --- a/scripts/check-changed-skills.test.sh +++ b/scripts/check-changed-skills.test.sh @@ -22,16 +22,8 @@ stage_libs() { # shellcheck source=test-git-helpers.sh . "$SELF_DIR/test-git-helpers.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" # A stub checker shared by every scenario: records each invocation (skill name + # forwarded env) to $CHECK_LOG, and FAILs iff the skill is named "bad". @@ -339,6 +331,4 @@ fi rm -rf "$r" rm -f "$STUB" -echo -echo "PASS=$PASS FAIL=$FAIL" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh index c261497359..0089698a7a 100755 --- a/scripts/check-changelog-parity.test.sh +++ b/scripts/check-changelog-parity.test.sh @@ -20,16 +20,8 @@ stage_libs() { # shellcheck source=test-git-helpers.sh . "$SELF_DIR/test-git-helpers.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" mk_repo() { local dir @@ -1197,5 +1189,4 @@ else fi rm -rf "$repo" -printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL" -((FAIL == 0)) +test_harness::report diff --git a/scripts/check-contract-slice-prune.test.sh b/scripts/check-contract-slice-prune.test.sh index 1089b4228d..5a7a146c0e 100755 --- a/scripts/check-contract-slice-prune.test.sh +++ b/scripts/check-contract-slice-prune.test.sh @@ -14,16 +14,8 @@ SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT="$SELF_DIR/check-contract-slice-prune.sh" PARSE_LIB="$SELF_DIR/../lib/parse-concern-value.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" # Fixture git commands must not spawn background maintenance. On git >= 2.46, # `git commit` and `git merge` fork `git maintenance run --auto --quiet @@ -471,6 +463,4 @@ repo="$(mk_repo)" if (($? == 2)); then ok "unknown mode exits 2"; else fail "unknown mode must exit 2"; fi rm -rf "$repo" -echo "" -echo "check-contract-slice-prune.test.sh: $PASS passed, $FAIL failed" -((FAIL == 0)) +test_harness::report diff --git a/scripts/check-cross-plugin-source-drift.test.sh b/scripts/check-cross-plugin-source-drift.test.sh index 0ec25bd36e..da3f7f3240 100755 --- a/scripts/check-cross-plugin-source-drift.test.sh +++ b/scripts/check-cross-plugin-source-drift.test.sh @@ -11,16 +11,8 @@ set -uo pipefail SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT="$SELF_DIR/check-cross-plugin-source-drift.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" # new_fixture → prints the path to a fresh /scripts + /plugins tree # with the script copied in, ready for callers to populate. @@ -179,6 +171,4 @@ else fi fi -echo -echo "PASS=$PASS FAIL=$FAIL" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/check-discriminating-test-skips.test.sh b/scripts/check-discriminating-test-skips.test.sh index e8f3bae677..3a8291c9db 100755 --- a/scripts/check-discriminating-test-skips.test.sh +++ b/scripts/check-discriminating-test-skips.test.sh @@ -12,16 +12,8 @@ REPO_ROOT="$(cd "$SELF_DIR/.." && pwd)" SCRIPT="$SELF_DIR/check-discriminating-test-skips.sh" HELPERS="$REPO_ROOT/plugins/source-control/scripts/test-helpers.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" new_fixture() { local dir @@ -169,5 +161,4 @@ else fi rm -f "$tmp_test" -printf '\n%d passed, %d failed\n' "$PASS" "$FAIL" -[[ $FAIL -eq 0 ]] || exit 1 +test_harness::report diff --git a/scripts/check-docs-only.test.sh b/scripts/check-docs-only.test.sh index b1ee710829..3c81e31c18 100755 --- a/scripts/check-docs-only.test.sh +++ b/scripts/check-docs-only.test.sh @@ -23,16 +23,8 @@ ALLOWLIST="$SELF_DIR/docs-only-paths.txt" # shellcheck source=test-git-helpers.sh . "$SELF_DIR/test-git-helpers.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" mk_repo() { local dir @@ -150,5 +142,4 @@ else fi rm -rf "$repo" "$gho" -printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL" -((FAIL == 0)) +test_harness::report diff --git a/scripts/check-fleet-audit-doc-grammar.test.sh b/scripts/check-fleet-audit-doc-grammar.test.sh index c44281a09f..44565dab91 100755 --- a/scripts/check-fleet-audit-doc-grammar.test.sh +++ b/scripts/check-fleet-audit-doc-grammar.test.sh @@ -7,16 +7,8 @@ set -uo pipefail SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT="$SELF_DIR/check-fleet-audit-doc-grammar.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" # Minimal stub collector: classifies the bare-positional probe and the no-scope # probe from argv / emptiness without touching git or the network. @@ -285,5 +277,4 @@ else fail "repository skill+parser fails --check" fi -echo "PASS=$PASS FAIL=$FAIL" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/check-fleet-finding-test-coverage.test.sh b/scripts/check-fleet-finding-test-coverage.test.sh index 3564151473..c18c9fb552 100755 --- a/scripts/check-fleet-finding-test-coverage.test.sh +++ b/scripts/check-fleet-finding-test-coverage.test.sh @@ -7,16 +7,8 @@ set -uo pipefail SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT="$SELF_DIR/check-fleet-finding-test-coverage.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" mk_tree() { local dir @@ -220,5 +212,4 @@ else fail "repository baseline fails --check" fi -echo "PASS=$PASS FAIL=$FAIL" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/check-hook-exec-form.test.sh b/scripts/check-hook-exec-form.test.sh index 9b24516a47..631cdf641d 100755 --- a/scripts/check-hook-exec-form.test.sh +++ b/scripts/check-hook-exec-form.test.sh @@ -11,16 +11,8 @@ SCRIPT="$SELF_DIR/check-hook-exec-form.sh" READER="$SELF_DIR/check-hook-exec-form-frontmatter.py" REQUIREMENTS="$SELF_DIR/../.github/requirements-ci.txt" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" new_fixture() { local dir @@ -968,6 +960,4 @@ else fi rm -rf "$f" -echo -echo "passed: $PASS, failed: $FAIL" -((FAIL == 0)) || exit 1 +test_harness::report diff --git a/scripts/check-hook-userconfig-argv.test.sh b/scripts/check-hook-userconfig-argv.test.sh index faaf9e1252..3018b3344d 100755 --- a/scripts/check-hook-userconfig-argv.test.sh +++ b/scripts/check-hook-userconfig-argv.test.sh @@ -18,16 +18,8 @@ stage_libs() { cp "$SELF_DIR/lib/read-list.sh" "$1/lib/" } -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" new_fixture() { local dir @@ -293,6 +285,4 @@ else fi rm -rf "$f" -echo -echo "passed: $PASS, failed: $FAIL" -((FAIL == 0)) || exit 1 +test_harness::report diff --git a/scripts/check-orphaned-fixtures.test.sh b/scripts/check-orphaned-fixtures.test.sh index b7b19860d1..8fb2300a93 100755 --- a/scripts/check-orphaned-fixtures.test.sh +++ b/scripts/check-orphaned-fixtures.test.sh @@ -17,16 +17,8 @@ stage_libs() { cp "$SELF_DIR/lib/read-list.sh" "$1/lib/" } -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" # mk_repo : throwaway repo with the detector installed and a # baseline file at scripts/orphaned-fixtures-baseline.txt. @@ -185,5 +177,4 @@ repo="$(mk_repo)" if [[ $? -eq 2 ]]; then ok "bad mode -> exit 2"; else fail "bad mode did not exit 2"; fi rm -rf "$repo" -printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL" -((FAIL == 0)) +test_harness::report diff --git a/scripts/check-queue-front-matter.test.sh b/scripts/check-queue-front-matter.test.sh index 3fe66ce51e..271032715f 100755 --- a/scripts/check-queue-front-matter.test.sh +++ b/scripts/check-queue-front-matter.test.sh @@ -5,16 +5,8 @@ set -uo pipefail SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT="$SELF_DIR/check-queue-front-matter.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" new_queue() { mktemp -d @@ -103,5 +95,4 @@ else fail "README should be ignored: valid sole item should pass" fi -printf '\n%d passed, %d failed\n' "$PASS" "$FAIL" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/check-shell-portability.test.sh b/scripts/check-shell-portability.test.sh index f4872cc3ea..1f81717121 100755 --- a/scripts/check-shell-portability.test.sh +++ b/scripts/check-shell-portability.test.sh @@ -14,9 +14,8 @@ # reference implementation, and the shipping list's staged-class inertness — # to prove the shipping config, not just the mechanism. # -# Bespoke PASS/FAIL counters by design, not drift: this is repo tooling, not a -# plugin, so no plugin assertion library applies here — see -# docs/conventions/shell-test-helpers/README.md. +# Assertion scaffolding lives in scripts/lib/test-harness.sh — repo tooling, +# not a plugin library. See docs/conventions/shell-test-helpers/README.md. # shellcheck disable=SC2016 # fixture bodies are literal shell content in single quotes; expansion is never wanted set -uo pipefail @@ -36,16 +35,8 @@ stage_libs() { REAL_TOKENS="$REPO_ROOT/scripts/shell-portability-tokens.txt" . "$SELF_DIR/test-git-helpers.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" # scan_paths ... — run the gate over explicit paths. scan_paths() { @@ -3257,6 +3248,4 @@ else fi rm -f "$f" -echo -echo "PASS=$PASS FAIL=$FAIL" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/check-silent-revert.test.sh b/scripts/check-silent-revert.test.sh index 9049671d4a..21e27f14de 100755 --- a/scripts/check-silent-revert.test.sh +++ b/scripts/check-silent-revert.test.sh @@ -23,16 +23,8 @@ SCRIPT="$SELF_DIR/check-silent-revert.sh" # shellcheck source=test-git-helpers.sh . "$SELF_DIR/test-git-helpers.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" TMPDIRS=() cleanup() { @@ -2374,6 +2366,4 @@ t_restoration_binds_a_marker_to_exactly_one_file t_restoration_and_replay_share_the_corpus t_shipped_data_files_are_wellformed -echo -echo "check-silent-revert.test.sh: $PASS passed, $FAIL failed" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/check-silent-skips.test.sh b/scripts/check-silent-skips.test.sh index 123770403b..b4cdf7fea2 100755 --- a/scripts/check-silent-skips.test.sh +++ b/scripts/check-silent-skips.test.sh @@ -9,16 +9,8 @@ set -uo pipefail SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT="$SELF_DIR/check-silent-skips.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" new_fixture() { local dir @@ -296,6 +288,4 @@ else fi rm -rf "$f" -echo -echo "PASS=$PASS FAIL=$FAIL" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/check-skill-portability.test.sh b/scripts/check-skill-portability.test.sh index 1f0ac822f7..a38b0c0cde 100755 --- a/scripts/check-skill-portability.test.sh +++ b/scripts/check-skill-portability.test.sh @@ -6,9 +6,8 @@ # silent-skip suite does. Two cases run against the REAL corpus to prove the # bare-vs-guarded discrimination on live files, not only synthetic ones. # -# Bespoke PASS/FAIL counters by design, not drift: this is repo tooling, not a -# plugin, so no plugin assertion library applies here — see -# docs/conventions/shell-test-helpers/README.md. +# Assertion scaffolding lives in scripts/lib/test-harness.sh — repo tooling, +# not a plugin library. See docs/conventions/shell-test-helpers/README.md. # shellcheck disable=SC2016 # fixture bodies are literal skill content in single quotes; expansion is never wanted set -uo pipefail @@ -34,16 +33,8 @@ TEST_TOKENS="$(mktemp)" printf 'origin/(main|master)\n' >"$TEST_TOKENS" trap 'rm -f "$TEST_TOKENS"' EXIT -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" # scan_paths ... — run the gate over explicit paths with the test tokens. scan_paths() { @@ -1037,6 +1028,4 @@ else ok "no ACTIVE token carries a backslash-b word-boundary escape" fi -echo -echo "PASS=$PASS FAIL=$FAIL" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/check-skill-precompute-compose.test.sh b/scripts/check-skill-precompute-compose.test.sh index 35e488653c..22d172eda6 100755 --- a/scripts/check-skill-precompute-compose.test.sh +++ b/scripts/check-skill-precompute-compose.test.sh @@ -7,16 +7,8 @@ SCRIPT="$SELF_DIR/check-skill-precompute-compose.sh" # shellcheck source=test-git-helpers.sh . "$SELF_DIR/test-git-helpers.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" new_fixture() { local dir @@ -195,5 +187,4 @@ else fi rm -rf "$f" -printf '\n%d passed, %d failed\n' "$PASS" "$FAIL" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/generate-cheatsheet.test.sh b/scripts/generate-cheatsheet.test.sh index 4e1b86e41f..fb5655715d 100755 --- a/scripts/generate-cheatsheet.test.sh +++ b/scripts/generate-cheatsheet.test.sh @@ -21,16 +21,8 @@ case "$REAL_SCRIPTS_DIR" in *) REAL_CONFIG_URL="file:///$REAL_SCRIPTS_DIR/cheatsheet-config.mjs" ;; esac -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" # write_skill [metadata lines...] write_skill() { @@ -276,6 +268,4 @@ printf '## 3. Blueprint (renamed upstream)\n' \ >"$tree/plugins/session-flow/skills/workflow/context/steps.md" assert_fails "spine-drift assertion fires on renamed stage" "$tree" "spine drift" -echo -echo "passed=$PASS failed=$FAIL" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/lib/changed-files.test.sh b/scripts/lib/changed-files.test.sh index d63d8d247b..cb7e3929dc 100755 --- a/scripts/lib/changed-files.test.sh +++ b/scripts/lib/changed-files.test.sh @@ -19,16 +19,8 @@ SCRIPTS_DIR="$(cd "$SELF_DIR/.." && pwd)" # shellcheck source=../test-git-helpers.sh . "$SCRIPTS_DIR/test-git-helpers.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=test-harness.sh +. "$SELF_DIR/test-harness.sh" # mk_repo -> prints the path of a fresh repo with one committed base tree. mk_repo() { @@ -271,8 +263,4 @@ else fi rm -rf "$repo" -# --- summary -------------------------------------------------------------- - -echo -echo "passed: $PASS failed: $FAIL" -((FAIL == 0)) || exit 1 +test_harness::report diff --git a/scripts/lib/read-list.test.sh b/scripts/lib/read-list.test.sh index f6e6d5badd..518aff7784 100755 --- a/scripts/lib/read-list.test.sh +++ b/scripts/lib/read-list.test.sh @@ -13,16 +13,8 @@ SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=read-list.sh . "$SELF_DIR/read-list.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=test-harness.sh +. "$SELF_DIR/test-harness.sh" # mk -> path of a temp list file mk() { @@ -163,6 +155,4 @@ else fail "the shipped docs-only allowlist yielded nothing" fi -echo -echo "passed: $PASS failed: $FAIL" -((FAIL == 0)) || exit 1 +test_harness::report diff --git a/scripts/lib/test-harness.sh b/scripts/lib/test-harness.sh new file mode 100644 index 0000000000..d2050003d1 --- /dev/null +++ b/scripts/lib/test-harness.sh @@ -0,0 +1,49 @@ +# shellcheck shell=bash +# Shared assertion scaffolding for scripts/*.test.sh. Sourced, never executed. +# +# The suites under scripts/ used to re-declare PASS/FAIL counters, ok()/fail(), +# and a trailing summary in eight spellings. The copies were free to disagree +# on the part that matters: a suite whose summary was wrong could print +# failures and still exit 0, and CI would treat that as green. One library +# owns the pass-recording / fail-recording / summary / exit-status contract. +# +# This is the called shape, not an EXIT trap. Bash runs exactly one EXIT trap +# per shell; an installed summary contended with the suites that already use +# that slot for fixture cleanup (check-skill-portability.test.sh, +# check-silent-revert.test.sh). test_harness::report is each suite's last +# line; the trap slot stays the caller's. A suite that exits before the call +# skips the verdict — the harness's own suite asserts that every sourcer ends +# with the call, so a migration that drops it cannot land quietly. +# +# Plugin *.test.sh helpers stay duplicated on purpose; see +# docs/conventions/shell-test-helpers/README.md. This file is the repo-tooling +# layer only. + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + printf 'scripts/lib/test-harness.sh is sourced-only\n' >&2 + exit 2 +fi + +_test_harness_pass=0 +_test_harness_fail=0 + +ok() { + printf 'ok: %s\n' "$*" + _test_harness_pass=$((_test_harness_pass + 1)) +} + +fail() { + printf 'FAIL: %s\n' "$*" >&2 + _test_harness_fail=$((_test_harness_fail + 1)) +} + +# test_harness::report +# Prints PASS/FAIL totals and returns 1 when any fail() was recorded, else 0. +# Invoke as the suite's last line so the return is the script's exit status. +test_harness::report() { + printf 'PASS=%d FAIL=%d\n' "$_test_harness_pass" "$_test_harness_fail" + if ((_test_harness_fail > 0)); then + return 1 + fi + return 0 +} diff --git a/scripts/lib/test-harness.test.sh b/scripts/lib/test-harness.test.sh new file mode 100755 index 0000000000..479133be5d --- /dev/null +++ b/scripts/lib/test-harness.test.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Self-test for scripts/lib/test-harness.sh. +# +# The load-bearing property is the exit contract: a suite that recorded a +# failed assertion and then called test_harness::report cannot exit 0. The +# other cases pin the print format, the sourced-only guard, and the +# last-line discipline the called shape depends on. +set -uo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +HARNESS="$SELF_DIR/test-harness.sh" +REPO_ROOT="$(cd "$SELF_DIR/../.." && pwd)" + +# shellcheck source=test-harness.sh +. "$HARNESS" + +run_child() { + local body="$1" + local out rc + out="$(bash -c "$body" 2>&1)" && rc=0 || rc=$? + printf '%s\n' "$out" + return "$rc" +} + +# --- sourced-only --------------------------------------------------------- + +if bash "$HARNESS" >/dev/null 2>&1; then + fail "executing the library exited 0" +else + rc=$? + if [[ "$rc" -eq 2 ]]; then + ok "executing the library exits 2" + else + fail "executing the library exited $rc, expected 2" + fi +fi + +# --- fail then report cannot exit 0 --------------------------------------- + +out="$( + run_child " + # shellcheck source=test-harness.sh + . \"$HARNESS\" + fail \"recorded failure\" + test_harness::report + " +)" && rc=0 || rc=$? +if [[ "$rc" -ne 0 && "$out" == *"FAIL: recorded failure"* && "$out" == *"PASS=0 FAIL=1"* ]]; then + ok "a recorded failure plus report exits non-zero" +else + fail "fail+report should exit non-zero with FAIL=1, got rc=$rc out='$out'" +fi + +# --- ok then report exits 0 ----------------------------------------------- + +out="$( + run_child " + . \"$HARNESS\" + ok \"one pass\" + test_harness::report + " +)" && rc=0 || rc=$? +if [[ "$rc" -eq 0 && "$out" == *"ok: one pass"* && "$out" == *"PASS=1 FAIL=0"* ]]; then + ok "an all-pass suite exits 0 with PASS=1 FAIL=0" +else + fail "ok+report should exit 0 with PASS=1 FAIL=0, got rc=$rc out='$out'" +fi + +# Mixed: one fail among passes still exits non-zero. +out="$( + run_child " + . \"$HARNESS\" + ok \"a\" + fail \"b\" + ok \"c\" + test_harness::report + " +)" && rc=0 || rc=$? +if [[ "$rc" -ne 0 && "$out" == *"PASS=2 FAIL=1"* ]]; then + ok "mixed pass/fail reports PASS=2 FAIL=1 and exits non-zero" +else + fail "mixed run should be PASS=2 FAIL=1 non-zero, got rc=$rc out='$out'" +fi + +# --- last-line discipline ------------------------------------------------- + +last_code_line() { + local f="$1" + awk ' + { gsub(/\r$/, "") } + /^[[:space:]]*#/ { next } + /^[[:space:]]*$/ { next } + { last=$0 } + END { print last } + ' "$f" +} + +sources_harness() { + grep -qE 'lib/test-harness\.sh|source=test-harness\.sh' "$1" +} + +shopt -s nullglob +missing=() +for f in "$REPO_ROOT"/scripts/*.test.sh "$REPO_ROOT"/scripts/lib/*.test.sh; do + sources_harness "$f" || continue + last="$(last_code_line "$f")" + if [[ "$last" != "test_harness::report" ]]; then + missing+=("${f#"$REPO_ROOT"/}:$last") + fi +done + +if ((${#missing[@]} == 0)); then + ok "every suite that sources the harness ends with test_harness::report" +else + fail "sourcer(s) missing trailing test_harness::report: ${missing[*]}" +fi + +test_harness::report diff --git a/scripts/sync-standards-contract.test.sh b/scripts/sync-standards-contract.test.sh index 7e1f14bc3a..b2c4647d09 100755 --- a/scripts/sync-standards-contract.test.sh +++ b/scripts/sync-standards-contract.test.sh @@ -10,16 +10,8 @@ SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" SCRIPT="$SELF_DIR/sync-standards-contract.sh" . "$SELF_DIR/test-git-helpers.sh" -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" CANONICAL="docs/conventions/standards/README.md" CHANGELOG="docs/conventions/standards/CHANGELOG.md" @@ -273,6 +265,4 @@ else fi rm -rf "$f" -echo -echo "PASS=$PASS FAIL=$FAIL" -[[ "$FAIL" -eq 0 ]] +test_harness::report diff --git a/scripts/test-git-helpers.test.sh b/scripts/test-git-helpers.test.sh index ff789d9c58..ed3dacc30f 100755 --- a/scripts/test-git-helpers.test.sh +++ b/scripts/test-git-helpers.test.sh @@ -63,16 +63,8 @@ if [[ ! -f "$HELPER" ]]; then exit 1 fi -PASS=0 -FAIL=0 -fail() { - echo "FAIL: $*" >&2 - FAIL=$((FAIL + 1)) -} -ok() { - echo "ok: $*" - PASS=$((PASS + 1)) -} +# shellcheck source=lib/test-harness.sh +. "$SELF_DIR/lib/test-harness.sh" SENTINEL="sentinel@example.invalid" @@ -231,6 +223,4 @@ scenario_c scenario_d scenario_declares_full_env_clear -echo -echo "passed: $PASS failed: $FAIL" -((FAIL == 0)) || exit 1 +test_harness::report