From f208709105d9368a8d7d03cc30d4db4872cf2db6 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 07:25:29 -0400 Subject: [PATCH 01/12] feat(ci): orphaned-fixture + CHANGELOG-parity gates (#663) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two repo-local CI gates from the 2026-07-20 merged-PR quality audit, following the scripts/ + ci.yml precedent of the docs-only and cross-plugin-source-drift gates: self-testable, fail-closed, stale-guarded baselines that grandfather existing debt without red-lining it. Gate 1 (orphaned-fixture): every file under a skill's evals/fixtures/ must be consumed by a grader — an eval files[] entry or a test assertion. Gate 2 (CHANGELOG-parity): a versioned plugin must keep a CHANGELOG.md (static), and a PR that bumps a plugin's version must update its CHANGELOG.md in the same diff (--check-bump). Co-Authored-By: Claude Sonnet 5 --- .github/workflows/ci.yml | 49 +++++++ scripts/changelog-parity-baseline.txt | 12 ++ scripts/check-changelog-parity.sh | 134 ++++++++++++++++++ scripts/check-changelog-parity.test.sh | 144 +++++++++++++++++++ scripts/check-orphaned-fixtures.sh | 179 ++++++++++++++++++++++++ scripts/check-orphaned-fixtures.test.sh | 104 ++++++++++++++ scripts/orphaned-fixtures-baseline.txt | 25 ++++ 7 files changed, 647 insertions(+) create mode 100644 scripts/changelog-parity-baseline.txt create mode 100755 scripts/check-changelog-parity.sh create mode 100755 scripts/check-changelog-parity.test.sh create mode 100755 scripts/check-orphaned-fixtures.sh create mode 100755 scripts/check-orphaned-fixtures.test.sh create mode 100644 scripts/orphaned-fixtures-baseline.txt diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daeb040fe..dcd906cdb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -295,6 +295,53 @@ jobs: - name: Run silent-skip-gate tests run: bash scripts/check-silent-skips.test.sh + # Every file under a skill's evals/fixtures/ must be consumed by a grader — an + # eval files[] entry or a test assertion — so an ungraded suite cannot sit in + # the tree reading as tested. The check is repo-wide and static; existing debt + # is grandfathered in scripts/orphaned-fixtures-baseline.txt with a stale + # guard, so the gate holds the line without red-lining fixtures a member issue + # already owns. The self-test runs unconditionally so a broken detector cannot + # mask a regression behind a green gate. + orphaned-fixture-gate: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Check out + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Test the orphaned-fixture detector + run: bash scripts/check-orphaned-fixtures.test.sh + - name: Check for orphaned eval fixtures + run: scripts/check-orphaned-fixtures.sh --check + + # CHANGELOG parity: a versioned plugin must keep a CHANGELOG.md (static + # --check), and a PR that changes a plugin's manifest version must update that + # plugin's CHANGELOG.md in the same diff (--check-bump, PR-only). Existing + # "versioned but changelog-less" debt is grandfathered by name in + # scripts/changelog-parity-baseline.txt with a stale guard; the bump gate is + # never relaxed by the baseline. The self-test runs unconditionally so a broken + # gate cannot mask a regression, and the PR-diff step is event-gated. + changelog-parity-gate: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Check out + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + # Full history so the PR base ref is resolvable for the bump gate. + fetch-depth: 0 + - name: Test the changelog-parity gate + run: bash scripts/check-changelog-parity.test.sh + - name: Verify every versioned plugin keeps a CHANGELOG.md + run: scripts/check-changelog-parity.sh --check + - name: Verify a version bump updates the plugin's CHANGELOG.md + if: github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.base_ref }} + run: scripts/check-changelog-parity.sh --check-bump "origin/$BASE_REF" + # Deep plugin-contract lane and the heaviest suite (Node + Python installs, # every plugins/**/*.test.sh, manifest + catalog validation). A PR whose diff # is confined to the docs-only allowlist (scripts/docs-only-paths.txt) cannot @@ -531,6 +578,8 @@ jobs: - standards-contract-sync - cross-plugin-source-drift - silent-skip-gate + - orphaned-fixture-gate + - changelog-parity-gate - plugin-gate - miro-plugin - runner-policy diff --git a/scripts/changelog-parity-baseline.txt b/scripts/changelog-parity-baseline.txt new file mode 100644 index 000000000..3898ba24c --- /dev/null +++ b/scripts/changelog-parity-baseline.txt @@ -0,0 +1,12 @@ +# Grandfathered "versioned plugin with no CHANGELOG.md" debt — existing gaps +# check-changelog-parity.sh --check must not red-line. One plugin NAME per line +# (comments and blank lines ignored). Stale-guarded: --check fails if a listed +# plugin gains a CHANGELOG.md or drops its version, so an entry cannot outlive +# its debt. This never relaxes --check-bump — a listed plugin that bumps its +# version still must start its changelog in that same PR. +# +# autonomy: shipped 5 minor version bumps with no CHANGELOG.md while every other +# plugin maintains one — the parity gap this gate exists to flag. Reconstructing +# an accurate history is autonomy-plugin (WP-lane) work, not this repo-level CI +# change; grandfathered pending a dedicated autonomy CHANGELOG issue. +autonomy diff --git a/scripts/check-changelog-parity.sh b/scripts/check-changelog-parity.sh new file mode 100755 index 000000000..2ab2154e8 --- /dev/null +++ b/scripts/check-changelog-parity.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# Keep each plugin's CHANGELOG.md honest against its manifest version. +# +# scripts/check-changelog-parity.sh --check fail if a versioned +# plugin has no sibling +# CHANGELOG.md +# scripts/check-changelog-parity.sh --check-bump fail if a plugin's +# manifest version +# changed vs but +# its CHANGELOG.md was +# not touched in the diff +# +# Two complementary gaps the same audit surfaced: +# * --check is the static repo-wide invariant: a plugins//.claude-plugin/ +# plugin.json carrying a `version` must ship a plugins//CHANGELOG.md. +# It catches a plugin that has bumped versions but never kept a changelog at +# all (autonomy shipped 5 minor bumps with none). +# * --check-bump is the go-forward PR discipline: a version change with no +# CHANGELOG.md edit in the same diff means the release is undocumented. It +# mirrors the sync-*.sh --check-bump bump gates and applies to EVERY plugin, +# grandfathered or not — the moment a debt-listed plugin bumps again it must +# start its changelog. +# +# Existing "versioned but changelog-less" debt is grandfathered by plugin NAME in +# scripts/changelog-parity-baseline.txt (same stale-guarded idiom as +# scripts/orphaned-fixtures-baseline.txt): --check exempts a listed plugin but +# fails on a STALE entry — one that now has a CHANGELOG.md (or no version) — so +# the exemption cannot outlive the debt. The baseline never relaxes --check-bump. +# +# Fail-closed: a versioned plugin with neither a CHANGELOG.md nor a baseline +# entry fails. CHANGELOG_PARITY_BASELINE overrides the baseline path (test +# injection). +set -uo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 2 + +BASELINE="${CHANGELOG_PARITY_BASELINE:-scripts/changelog-parity-baseline.txt}" + +mode="${1:-}" +case "$mode" in +--check | --check-bump) ;; +*) + echo "usage: $(basename "$0") [--check | --check-bump ]" >&2 + exit 2 + ;; +esac + +# Grandfathered plugin names (static-check exemptions). +declare -A grandfathered +if [[ -f "$BASELINE" ]]; then + while IFS= read -r line; do + line="${line%%#*}" + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ -z "$line" ]] && continue + grandfathered["$line"]=1 + done <"$BASELINE" +fi + +manifests=(plugins/*/.claude-plugin/plugin.json) +if [[ ! -e "${manifests[0]}" ]]; then + echo "error: no plugin manifests found under plugins/*/.claude-plugin/" >&2 + exit 2 +fi + +version_of() { jq -r '.version // empty' "$1" 2>/dev/null; } + +if [[ "$mode" == "--check" ]]; then + declare -A saw_debt + missing=0 + for manifest in "${manifests[@]}"; do + plugin_dir="${manifest%/.claude-plugin/plugin.json}" + name="${plugin_dir##*/}" + [[ -n "$(version_of "$manifest")" ]] || continue + if [[ -f "$plugin_dir/CHANGELOG.md" ]]; then + if [[ -n "${grandfathered[$name]:-}" ]]; then + echo "STALE BASELINE: '$name' in $BASELINE now has a CHANGELOG.md — remove it." >&2 + missing=$((missing + 1)) + fi + continue + fi + if [[ -n "${grandfathered[$name]:-}" ]]; then + saw_debt["$name"]=1 + continue + fi + echo "MISSING CHANGELOG: $plugin_dir carries a versioned $manifest but no $plugin_dir/CHANGELOG.md." >&2 + echo " Add $plugin_dir/CHANGELOG.md, or grandfather '$name' in $BASELINE with its owning issue." >&2 + missing=$((missing + 1)) + done + # A baseline name that matches no versioned-and-changelog-less plugin is stale. + for name in "${!grandfathered[@]}"; do + if [[ -z "${saw_debt[$name]:-}" ]]; then + echo "STALE BASELINE: '$name' in $BASELINE no longer names a versioned plugin missing a CHANGELOG.md — remove it." >&2 + missing=$((missing + 1)) + fi + done + if ((missing > 0)); then + exit 1 + fi + echo "Every versioned plugin has a CHANGELOG.md (or a stale-guarded baseline entry)." + exit 0 +fi + +# --check-bump mode +base="${2:?usage: check-changelog-parity.sh --check-bump }" +if ! git rev-parse --verify --quiet "${base}^{commit}" >/dev/null; then + echo "check-changelog-parity: base ref '$base' is not a resolvable commit." >&2 + exit 2 +fi + +undocumented=0 +for manifest in "${manifests[@]}"; do + plugin_dir="${manifest%/.claude-plugin/plugin.json}" + name="${plugin_dir##*/}" + changelog="$plugin_dir/CHANGELOG.md" + + base_version="$(git show "$base:$manifest" 2>/dev/null | jq -r '.version // empty' 2>/dev/null || true)" + # Absent at base => new plugin in this change set; the static --check owns + # whether its initial CHANGELOG.md exists, not the bump gate. + [[ -n "$base_version" ]] || continue + head_version="$(version_of "$manifest")" + [[ "$head_version" != "$base_version" ]] || continue + + if git diff --quiet "$base" -- "$changelog"; then + echo "UNDOCUMENTED BUMP: $name went $base_version -> $head_version but $changelog was not updated in this diff." >&2 + undocumented=$((undocumented + 1)) + fi +done + +if ((undocumented > 0)); then + echo "Add a CHANGELOG.md entry for every plugin whose version changed." >&2 + exit 1 +fi +echo "Every plugin whose version changed vs $base also updated its CHANGELOG.md." diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh new file mode 100755 index 000000000..fd62db25c --- /dev/null +++ b/scripts/check-changelog-parity.test.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Unit tests for check-changelog-parity.sh. --check scenarios build a throwaway +# plugins/ tree; --check-bump scenarios build a real two-commit git history so +# the version-changed-but-changelog-untouched case is exercised end to end. The +# negative cases are the synthetic proof the gate catches an undocumented +# version bump and a versioned-but-changelog-less plugin. +set -uo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="$SELF_DIR/check-changelog-parity.sh" + +PASS=0 +FAIL=0 +fail() { + echo "FAIL: $*" >&2 + FAIL=$((FAIL + 1)) +} +ok() { + echo "ok: $*" + PASS=$((PASS + 1)) +} + +mk_repo() { + local dir + dir="$(mktemp -d)" + mkdir -p "$dir/scripts" + cp "$SCRIPT" "$dir/scripts/check-changelog-parity.sh" + printf '%s' "${1:-}" >"$dir/scripts/changelog-parity-baseline.txt" + printf '%s' "$dir" +} + +git_init() { + local dir="$1" + git -C "$dir" init -q + git -C "$dir" config user.email t@t.test + git -C "$dir" config user.name test + git -C "$dir" config commit.gpgsign false + git -C "$dir" config core.autocrlf false +} + +mk_plugin() { + local repo="$1" name="$2" version="$3" changelog="$4" + mkdir -p "$repo/plugins/$name/.claude-plugin" + printf '{ "name": "%s", "version": "%s" }\n' "$name" "$version" \ + >"$repo/plugins/$name/.claude-plugin/plugin.json" + [[ "$changelog" == "yes" ]] && printf '# Changelog\n' >"$repo/plugins/$name/CHANGELOG.md" +} + +# ============================ --check (static) ============================= + +# versioned plugin WITH a changelog -> passes +repo="$(mk_repo)" +mk_plugin "$repo" alpha 1.0.0 yes +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check >/dev/null 2>&1); then ok "versioned plugin with CHANGELOG passes --check"; else fail "versioned+changelog wrongly failed"; fi +rm -rf "$repo" + +# SYNTHETIC MISSING CHANGELOG: versioned plugin, no changelog -> fails +repo="$(mk_repo)" +mk_plugin "$repo" alpha 1.0.0 no +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"MISSING CHANGELOG"*"alpha"* ]]; then ok "versioned plugin without CHANGELOG fails --check (synthetic gap caught)"; else fail "missing-changelog not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# grandfathered plugin without a changelog -> passes +repo="$(mk_repo $'alpha\n')" +mk_plugin "$repo" alpha 1.0.0 no +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check >/dev/null 2>&1); then ok "grandfathered missing-changelog passes --check"; else fail "grandfathered plugin wrongly failed"; fi +rm -rf "$repo" + +# STALE baseline: grandfathered plugin that DOES have a changelog -> fails +repo="$(mk_repo $'alpha\n')" +mk_plugin "$repo" alpha 1.0.0 yes +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"STALE BASELINE"*"alpha"* ]]; then ok "stale baseline entry fails --check"; else fail "stale baseline not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# ============================ --check-bump (diff) ========================= + +# version changed AND changelog updated -> passes +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n## 1.1.0\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "bump + changelog edit passes --check-bump"; else fail "bump+changelog wrongly failed"; fi +rm -rf "$repo" + +# SYNTHETIC UNDOCUMENTED BUMP: version changed, changelog untouched -> fails +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* ]]; then ok "bump without changelog fails --check-bump (synthetic undocumented bump caught)"; else fail "undocumented bump not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# version unchanged -> passes (no bump, nothing to document) +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf 'unrelated\n' >"$repo/plugins/alpha/README.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm noop +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "unchanged version passes --check-bump"; else fail "unchanged version wrongly failed"; fi +rm -rf "$repo" + +# new plugin absent at base -> --check-bump skips it (static --check owns it) +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +mk_plugin "$repo" beta 1.0.0 no +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm add-beta +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "new plugin skipped by --check-bump"; else fail "new plugin wrongly failed --check-bump"; fi +rm -rf "$repo" + +# unresolvable base ref -> exit 2 +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump does-not-exist >/dev/null 2>&1) +if [[ $? -eq 2 ]]; then ok "unresolvable base ref -> exit 2"; else fail "bad base ref did not exit 2"; fi +rm -rf "$repo" + +# bad usage -> exit 2 +repo="$(mk_repo)" +(cd "$repo" && bash scripts/check-changelog-parity.sh --nonsense >/dev/null 2>&1) +if [[ $? -eq 2 ]]; then ok "bad mode -> exit 2"; else fail "bad mode did not exit 2"; fi +rm -rf "$repo" + +printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL" +((FAIL == 0)) diff --git a/scripts/check-orphaned-fixtures.sh b/scripts/check-orphaned-fixtures.sh new file mode 100755 index 000000000..dd8bb07ab --- /dev/null +++ b/scripts/check-orphaned-fixtures.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Detect eval fixtures that no grader consumes: a file under a skill's +# evals/fixtures/ that no eval case references and no test asserts on. An +# ungraded fixture is dead weight that reads as tested — the "looks-tested" +# trap a merged-PR audit caught (autonomy shipped a large security-binding +# fixture corpus with no eval or test consuming any of it). +# +# scripts/check-orphaned-fixtures.sh discover: list every fixture +# under **/evals/fixtures/ and +# whether a grader consumes it +# scripts/check-orphaned-fixtures.sh --check fail if an un-grandfathered +# fixture is orphaned, or a +# baseline prefix is now stale +# +# A fixture at /evals/fixtures/ is CONSUMED when any of: +# * its skill-relative path (evals/fixtures/) appears in the sibling +# grader /evals/evals.json — an eval `files[]` entry, the issue's +# primary consumption path; +# * its basename appears as a whole token in that evals.json (a files[] form +# that spells the path differently still names the file); +# * its basename appears as a whole token in any *.test.* file in the plugin +# — a test that asserts on the fixture is a grader too. +# Consumption matching is deliberately generous (path OR basename, across the +# grader and every test): a false "consumed" only under-reports one orphan, +# whereas a false "orphaned" red-lines a legitimate fixture. The baseline +# absorbs the remaining known-orphan debt. +# +# Scope is **/evals/fixtures/ specifically — eval-grader fixtures, the gate's +# target. Unit-test fixture dirs (…/tests/fixtures, …/scripts/fixtures) are out +# of scope: those are often generated or loaded by directory, not named, and a +# name-matcher cannot honestly grade them. +# +# Existing orphan debt owned by another issue is grandfathered in +# scripts/orphaned-fixtures-baseline.txt (path-prefix lines, same idiom as +# scripts/docs-only-paths.txt). A prefix there is a promise the owning issue +# burns it down; --check fails on a stale prefix (one that no longer shadows any +# orphan) so the baseline cannot outlive its debt. +# +# Fail-closed: a fixture with no resolvable grader is an orphan unless +# explicitly grandfathered. FIXTURES_BASELINE overrides the baseline path (test +# injection). +set -uo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 2 + +BASELINE="${FIXTURES_BASELINE:-scripts/orphaned-fixtures-baseline.txt}" + +mode="${1:-discover}" +case "$mode" in +discover | --check) ;; +*) + echo "usage: $(basename "$0") [--check]" >&2 + exit 2 + ;; +esac + +# Active baseline prefixes: strip inline comments and surrounding blank lines. +prefixes=() +if [[ -f "$BASELINE" ]]; then + mapfile -t prefixes < <(sed -E 's/#.*//; s/^[[:space:]]+//; s/[[:space:]]+$//' "$BASELINE" | grep -v '^$') +fi + +is_grandfathered() { + local path="$1" prefix + for prefix in "${prefixes[@]}"; do + if [[ "$path" == "$prefix"* ]]; then + return 0 + fi + done + return 1 +} + +# plugin root that owns a path under plugins//… -> plugins/ +plugin_root_of() { + local path="$1" + # plugins / / rest… -> keep the first two segments + printf 'plugins/%s' "${path#plugins/}" | cut -d/ -f1-2 +} + +# consumed -> 0 if some grader consumes it +consumed() { + local fixture="$1" + local fixtures_dir evals_dir skill_dir evals_json rel base plugin + + fixtures_dir="${fixture%/*}" + # walk up to the nearest 'fixtures' segment (fixtures may nest a subdir) + while [[ "$fixtures_dir" == */* && "${fixtures_dir##*/}" != "fixtures" ]]; do + fixtures_dir="${fixtures_dir%/*}" + done + evals_dir="${fixtures_dir%/fixtures}" # …/evals + skill_dir="${evals_dir%/evals}" # dir that owns evals/ + evals_json="$evals_dir/evals.json" + rel="${fixture#"$skill_dir"/}" # evals/fixtures/ + base="${fixture##*/}" + + if [[ -f "$evals_json" ]]; then + if grep -qF "$rel" "$evals_json"; then + return 0 + fi + if grep -qwF "$base" "$evals_json"; then + return 0 + fi + fi + + plugin="$(plugin_root_of "$fixture")" + while IFS= read -r -d '' test_file; do + if grep -qwF "$base" "$test_file"; then + return 0 + fi + done < <(find "$plugin" -type f -name '*.test.*' -print0 2>/dev/null) + + return 1 +} + +# Collect every fixture under a **/evals/fixtures/ directory. +fixtures=() +while IFS= read -r -d '' dir; do + while IFS= read -r -d '' f; do + fixtures+=("$f") + done < <(find "$dir" -type f -print0) +done < <(find plugins -type d -path '*/evals/fixtures' -print0 2>/dev/null) + +if ((${#fixtures[@]} > 0)); then + mapfile -t fixtures < <(printf '%s\n' "${fixtures[@]}" | sort) +fi + +# Track which baseline prefixes actually shadow an orphan, to flag stale ones. +declare -A prefix_used +matched_prefix() { + local path="$1" prefix + for prefix in "${prefixes[@]}"; do + if [[ "$path" == "$prefix"* ]]; then + printf '%s' "$prefix" + return 0 + fi + done + return 1 +} + +orphans=0 +if [[ "$mode" == "discover" ]]; then + for f in "${fixtures[@]}"; do + if consumed "$f"; then + printf '%-14s %s\n' "CONSUMED" "$f" + elif is_grandfathered "$f"; then + printf '%-14s %s\n' "GRANDFATHERED" "$f" + else + printf '%-14s %s\n' "ORPHAN" "$f" + fi + done + exit 0 +fi + +# --check mode +for f in "${fixtures[@]}"; do + if consumed "$f"; then + continue + fi + if p="$(matched_prefix "$f")"; then + prefix_used["$p"]=1 + continue + fi + echo "ORPHANED FIXTURE: $f is under evals/fixtures/ but no eval case references it and no test asserts on it." >&2 + echo " Reference it from a grader (an eval files[] entry or a test), delete it, or grandfather it in $BASELINE with the owning issue." >&2 + orphans=$((orphans + 1)) +done + +stale=0 +for prefix in "${prefixes[@]}"; do + if [[ -z "${prefix_used[$prefix]:-}" ]]; then + echo "STALE BASELINE: '$prefix' in $BASELINE no longer shadows any orphaned fixture — remove it." >&2 + stale=$((stale + 1)) + fi +done + +if ((orphans > 0 || stale > 0)); then + exit 1 +fi +echo "No orphaned eval fixtures (every file under **/evals/fixtures/ is consumed by a grader or grandfathered)." diff --git a/scripts/check-orphaned-fixtures.test.sh b/scripts/check-orphaned-fixtures.test.sh new file mode 100755 index 000000000..fba069157 --- /dev/null +++ b/scripts/check-orphaned-fixtures.test.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# Unit tests for check-orphaned-fixtures.sh. Each scenario builds a throwaway +# plugins/ tree, runs the detector, and asserts the outcome. The negative cases +# (a fixture no grader consumes) are the synthetic proof the gate catches the +# "looks-tested but graded by nothing" class it exists for. +set -uo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="$SELF_DIR/check-orphaned-fixtures.sh" + +PASS=0 +FAIL=0 +fail() { + echo "FAIL: $*" >&2 + FAIL=$((FAIL + 1)) +} +ok() { + echo "ok: $*" + PASS=$((PASS + 1)) +} + +# mk_repo : throwaway repo with the detector installed and a +# baseline file at scripts/orphaned-fixtures-baseline.txt. +mk_repo() { + local dir + dir="$(mktemp -d)" + mkdir -p "$dir/scripts" + cp "$SCRIPT" "$dir/scripts/check-orphaned-fixtures.sh" + printf '%s' "${1:-}" >"$dir/scripts/orphaned-fixtures-baseline.txt" + printf '%s' "$dir" +} + +# seed a skill's evals.json with an optional files[] entry +seed_skill() { + local repo="$1" skill="$2" files_entry="$3" + mkdir -p "$repo/$skill/evals/fixtures" + cat >"$repo/$skill/evals/evals.json" </dev/null) +} + +# --- consumed via an eval files[] entry -> not an orphan -------------------- +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '"evals/fixtures/used.md"' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/used.md" +if run_check "$repo" >/dev/null; then ok "files[]-referenced fixture passes --check"; else fail "files[]-referenced fixture wrongly flagged"; fi +rm -rf "$repo" + +# --- consumed via a test file (basename) -> not an orphan ------------------ +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/by-test.md" +mkdir -p "$repo/plugins/p/skills/s/scripts" +printf 'assert_on fixtures/by-test.md\n' >"$repo/plugins/p/skills/s/scripts/thing.test.sh" +if run_check "$repo" >/dev/null; then ok "test-asserted fixture passes --check"; else fail "test-asserted fixture wrongly flagged"; fi +rm -rf "$repo" + +# --- SYNTHETIC ORPHAN: consumed by nothing -> --check fails ---------------- +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/orphan.md" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"orphan.md"* ]]; then ok "un-consumed fixture fails --check (synthetic orphan caught)"; else fail "synthetic orphan not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# --- orphan grandfathered by a baseline prefix -> passes ------------------- +repo="$(mk_repo $'plugins/p/skills/s/evals/fixtures/\n')" +seed_skill "$repo" "plugins/p/skills/s" '' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/orphan.md" +if run_check "$repo" >/dev/null; then ok "grandfathered orphan passes --check"; else fail "grandfathered orphan wrongly failed"; fi +rm -rf "$repo" + +# --- STALE baseline prefix (shadows no orphan) -> fails -------------------- +repo="$(mk_repo $'plugins/p/skills/s/evals/fixtures/\n')" +seed_skill "$repo" "plugins/p/skills/s" '"evals/fixtures/used.md"' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/used.md" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"STALE BASELINE"* ]]; then ok "stale baseline prefix fails --check"; else fail "stale baseline not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# --- discover mode labels every fixture ------------------------------------ +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '"evals/fixtures/used.md"' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/used.md" +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/orphan.md" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh discover 2>/dev/null)" +if [[ "$out" == *"CONSUMED"*"used.md"* && "$out" == *"ORPHAN"*"orphan.md"* ]]; then ok "discover labels CONSUMED and ORPHAN"; else fail "discover mislabels: '$out'"; fi +rm -rf "$repo" + +# --- bad usage -> exit 2 --------------------------------------------------- +repo="$(mk_repo)" +(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --nonsense >/dev/null 2>&1) +if [[ $? -eq 2 ]]; then ok "bad mode -> exit 2"; else fail "bad mode did not exit 2"; fi +rm -rf "$repo" + +printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL" +((FAIL == 0)) diff --git a/scripts/orphaned-fixtures-baseline.txt b/scripts/orphaned-fixtures-baseline.txt new file mode 100644 index 000000000..2481e04d8 --- /dev/null +++ b/scripts/orphaned-fixtures-baseline.txt @@ -0,0 +1,25 @@ +# Grandfathered orphaned eval fixtures — existing debt this gate must not +# red-line, each owned by (or awaiting) a burn-down issue. Path-prefix lines, +# same matching idiom as scripts/docs-only-paths.txt: a fixture whose path +# starts with an active line is exempt from check-orphaned-fixtures.sh --check. +# +# A prefix here is a standing promise the debt is tracked, not a place to hide +# new orphans: --check fails on a STALE prefix (one that no longer shadows any +# orphan), so an entry cannot outlive its debt. Delete the line when the owning +# issue lands the fix. Do NOT add a plugin's fixtures here to dodge the gate on +# NEW work — reference them from a grader instead. + +# autonomy setup skill: the security-binding golden suite (referenced by no +# eval files[] entry and graded by no test) plus the otlp-demo sample corpus. +# The security-binding subset is tracked by issue #662 (attach expected-verdict +# blocks and wire them into the grader, or demote them to documentation +# exemplars out of evals/fixtures/); otlp-demo is the same ungraded class in the +# same skill. autonomy is WP-lane-owned — the fix is coordinated there, not here. +plugins/autonomy/skills/setup/evals/fixtures/ + +# knowledge youtube-digest skill: this data file lives under evals/fixtures/ but +# is named only from SKILL.md / vendor TUNING.md prose — no eval files[] entry +# and no test consume it. Surfaced by this gate's initial discover run; it has +# no dedicated owning issue yet and needs triage (wire it into a grader, or move +# it out of evals/fixtures/ if it is reference data rather than a fixture). +plugins/knowledge/skills/youtube-digest/evals/fixtures/variation-matrix-backlog.json From e03985e90c20e36553ee464ee4ab4bf4e214d082 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 07:34:03 -0400 Subject: [PATCH 02/12] =?UTF-8?q?fix(ci):=20address=20review=20=E2=80=94?= =?UTF-8?q?=20dedupe=20stale=20message,=20exit-2=20on=20missing=20bump=20a?= =?UTF-8?q?rg,=20null-safe=20sort?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolves the claude review-bot findings on #681: - check-changelog-parity.sh --check: mark a grandfathered plugin that gained a CHANGELOG.md as handled so the second stale-scan loop no longer double-reports it. - check-changelog-parity.sh --check-bump: explicit missing-arg guard exiting 2, consistent with every other usage-error path (${2:?} exited 1). - check-orphaned-fixtures.sh: null-safe fixture sort (sort -z / mapfile -d ''). - Tests: assert the stale message appears exactly once; cover --check-bump with no base ref -> exit 2. Co-Authored-By: Claude Sonnet 5 --- scripts/check-changelog-parity.sh | 8 +++++++- scripts/check-changelog-parity.test.sh | 10 +++++++++- scripts/check-orphaned-fixtures.sh | 2 +- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/scripts/check-changelog-parity.sh b/scripts/check-changelog-parity.sh index 2ab2154e8..72aed4f55 100755 --- a/scripts/check-changelog-parity.sh +++ b/scripts/check-changelog-parity.sh @@ -76,6 +76,8 @@ if [[ "$mode" == "--check" ]]; then if [[ -n "${grandfathered[$name]:-}" ]]; then echo "STALE BASELINE: '$name' in $BASELINE now has a CHANGELOG.md — remove it." >&2 missing=$((missing + 1)) + # Mark handled so the second stale-scan loop does not re-report it. + saw_debt["$name"]=1 fi continue fi @@ -102,7 +104,11 @@ if [[ "$mode" == "--check" ]]; then fi # --check-bump mode -base="${2:?usage: check-changelog-parity.sh --check-bump }" +if [[ -z "${2:-}" ]]; then + echo "usage: $(basename "$0") --check-bump " >&2 + exit 2 +fi +base="$2" if ! git rev-parse --verify --quiet "${base}^{commit}" >/dev/null; then echo "check-changelog-parity: base ref '$base' is not a resolvable commit." >&2 exit 2 diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh index fd62db25c..bd4333bd1 100755 --- a/scripts/check-changelog-parity.test.sh +++ b/scripts/check-changelog-parity.test.sh @@ -73,7 +73,8 @@ repo="$(mk_repo $'alpha\n')" mk_plugin "$repo" alpha 1.0.0 yes out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check 2>&1)" rc=$? -if [[ $rc -ne 0 && "$out" == *"STALE BASELINE"*"alpha"* ]]; then ok "stale baseline entry fails --check"; else fail "stale baseline not caught: rc=$rc out='$out'"; fi +stale_count="$(printf '%s\n' "$out" | grep -c 'STALE BASELINE')" +if [[ $rc -ne 0 && "$stale_count" == "1" ]]; then ok "stale baseline entry fails --check (reported exactly once)"; else fail "stale baseline: rc=$rc count=$stale_count out='$out'"; fi rm -rf "$repo" # ============================ --check-bump (diff) ========================= @@ -134,6 +135,13 @@ git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base if [[ $? -eq 2 ]]; then ok "unresolvable base ref -> exit 2"; else fail "bad base ref did not exit 2"; fi rm -rf "$repo" +# --check-bump with no base ref -> exit 2 (consistent with other usage errors) +repo="$(mk_repo)" +mk_plugin "$repo" alpha 1.0.0 yes +(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump >/dev/null 2>&1) +if [[ $? -eq 2 ]]; then ok "--check-bump without base ref -> exit 2"; else fail "missing base ref did not exit 2"; fi +rm -rf "$repo" + # bad usage -> exit 2 repo="$(mk_repo)" (cd "$repo" && bash scripts/check-changelog-parity.sh --nonsense >/dev/null 2>&1) diff --git a/scripts/check-orphaned-fixtures.sh b/scripts/check-orphaned-fixtures.sh index dd8bb07ab..3e8ac172b 100755 --- a/scripts/check-orphaned-fixtures.sh +++ b/scripts/check-orphaned-fixtures.sh @@ -121,7 +121,7 @@ while IFS= read -r -d '' dir; do done < <(find plugins -type d -path '*/evals/fixtures' -print0 2>/dev/null) if ((${#fixtures[@]} > 0)); then - mapfile -t fixtures < <(printf '%s\n' "${fixtures[@]}" | sort) + mapfile -t -d '' fixtures < <(printf '%s\0' "${fixtures[@]}" | sort -z) fi # Track which baseline prefixes actually shadow an orphan, to flag stale ones. From e822423f03799bb3e13993ffbf38d1aefd398da5 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 08:36:55 -0400 Subject: [PATCH 03/12] fix(ci): pin orphaned-fixture baseline to exact files, not a directory prefix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The autonomy entry grandfathered a whole directory (plugins/autonomy/skills/setup/evals/fixtures/), so any NEW orphaned fixture later added under it would match the prefix and be silently exempted — and because that directory already shadows many current orphans, the prefix never goes stale to flag the drift. The gate could pass wrongly (fail-open): a new ungraded fixture in this skill bypasses the check entirely. Replace the single directory prefix with an exact-path snapshot of the 173 currently-known orphans (otlp-demo + security-binding), same prefix-matching idiom as scripts/docs-only-paths.txt but each line pins one file. A new orphan added under an already-listed directory now matches no line and is red-lined; staleness still fires per-file when an entry's file is consumed or deleted. The knowledge youtube-digest entry was already an exact file and is unchanged. Verified at head e03985e: the enumerated set equals a fresh empty-baseline discover (174 = 173 autonomy + 1 knowledge); --check on the real tree passes; a probe fixture dropped under security-binding/ is now caught (exit 1) where the directory prefix would have exempted it; the script's own test suite stays green (7/7, including the stale-entry case). Co-Authored-By: Claude Fable 5 --- scripts/orphaned-fixtures-baseline.txt | 200 +++++++++++++++++++++++-- 1 file changed, 188 insertions(+), 12 deletions(-) diff --git a/scripts/orphaned-fixtures-baseline.txt b/scripts/orphaned-fixtures-baseline.txt index 2481e04d8..dfe0148fa 100644 --- a/scripts/orphaned-fixtures-baseline.txt +++ b/scripts/orphaned-fixtures-baseline.txt @@ -1,21 +1,197 @@ # Grandfathered orphaned eval fixtures — existing debt this gate must not -# red-line, each owned by (or awaiting) a burn-down issue. Path-prefix lines, -# same matching idiom as scripts/docs-only-paths.txt: a fixture whose path -# starts with an active line is exempt from check-orphaned-fixtures.sh --check. +# red-line, each owned by (or awaiting) a burn-down issue. Exact-path lines, +# same matching idiom as scripts/docs-only-paths.txt (a fixture whose path +# starts with an active line is exempt from check-orphaned-fixtures.sh --check), +# but each line pins one file, never a directory: this is a snapshot of today's +# known orphans, so a NEW orphan added under an already-listed directory matches +# no line and is red-lined by the gate rather than silently grandfathered. # -# A prefix here is a standing promise the debt is tracked, not a place to hide -# new orphans: --check fails on a STALE prefix (one that no longer shadows any -# orphan), so an entry cannot outlive its debt. Delete the line when the owning +# An entry here is a standing promise the debt is tracked, not a place to hide +# new orphans: --check fails on a STALE entry (one that no longer shadows any +# orphan), so a line cannot outlive its debt. Delete the line when the owning # issue lands the fix. Do NOT add a plugin's fixtures here to dodge the gate on # NEW work — reference them from a grader instead. # autonomy setup skill: the security-binding golden suite (referenced by no -# eval files[] entry and graded by no test) plus the otlp-demo sample corpus. -# The security-binding subset is tracked by issue #662 (attach expected-verdict -# blocks and wire them into the grader, or demote them to documentation -# exemplars out of evals/fixtures/); otlp-demo is the same ungraded class in the -# same skill. autonomy is WP-lane-owned — the fix is coordinated there, not here. -plugins/autonomy/skills/setup/evals/fixtures/ +# eval files[] entry and graded by no test) plus the otlp-demo sample corpus, +# enumerated file-by-file below. The security-binding subset is tracked by +# issue #662 (attach expected-verdict blocks and wire them into the grader, or +# demote them to documentation exemplars out of evals/fixtures/); otlp-demo is +# the same ungraded class in the same skill. autonomy is WP-lane-owned — the fix +# is coordinated there, not here. +plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/pipeline.jsonl +plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/session-metrics.jsonl +plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/session-spans.jsonl +plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/trigger.jsonl +plugins/autonomy/skills/setup/evals/fixtures/security-binding/admission-weakened-no-justification.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ambiguous-runtime-markers.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/as112-v6-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/c2-auto-unratified.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/c3-blocking-unratified.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/c4-auto-merge.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/equal-specificity-tie.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/evidence-contrary.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/evidence-stale.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/hosted-ephemeral-l3-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/human-gated-only-no-l2.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/l3-ratified-class-container.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/marker-prototype-key.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/missing-l2-autonomous.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/missing-probe-evidence.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/missing-substrate-class.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/multi-host-egress-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-6to4-relay.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-6to4-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-bare-metadata.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-benchmarking-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-bogus-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-bogus-metadata.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-cred-outer-missing.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-cred-outer-nonzero.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-credpath-code-count.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-credpath-partial-zero.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-discard-only.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-doc-3fff.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-docnet-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-dot-traversal.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-dummy-prefix.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-encoded-loopback.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-etc-descendant-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-expanded-loopback.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-expansion-ephemeral.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-expansion-tail-mismatch.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-generic-env-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-generic-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-home-descendant-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-host-home-mount.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-imds-http-error.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invalid-tld.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invented-env-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invented-home-user.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invented-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-ipv4-compat.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-known-hosts.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-l3-container.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-level-mismatch.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-local-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-malformed-dns.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-port.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-route-boundary.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-route.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-scheme.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-host-expanded.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-outer-exit.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-probed-at.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-target.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-transport.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-multi-host-private.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-nat64-local.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-nat64-private-embed.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-outer-egress-failed.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-protocol-assignment.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-ratified-class-mismatch.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-relative-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-root-escape.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-site-local-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-srv6-sid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-ssh-dir.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-teredo.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-token-substring.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-transport-cross-type.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-unanchored-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-unanchored-envvar.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-unc-etc.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-uri-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-zero-exit.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-6to4-relay.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-6to4-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-as112-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-bare-metadata.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-benchmarking-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-bogus-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-bogus-metadata.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-cred-outer-missing.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-cred-outer-nonzero.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-credpath-code-count.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-credpath-partial-zero.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-discard-only.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-doc-3fff.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-docnet.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-dot-traversal.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-dummy-prefix.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-encoded-loopback.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-etc-descendant-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-expanded-loopback.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-expansion-ephemeral.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-expansion-tail-mismatch.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-generic-env-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-generic-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-home-descendant-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-host-home-mount.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-id-rsa.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-imds-http-error.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invalid-tld.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invented-env-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invented-home-user.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invented-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-ipv4-compat.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-known-hosts.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-localhost.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-malformed-dns.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-port.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-route-boundary.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-route.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-scheme.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-multi-host-private.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-multi-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-nat64-local.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-nat64-private-embed.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-host-expanded.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-outer-exit.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-probed-at.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-transport.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-outer-egress-failed.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-protocol-assignment.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-relative-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-site-local-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-srv6-sid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-ssh-dir.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-teredo.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-token-substring.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-transport-cross-type.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-unanchored-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-unanchored-envvar.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-unc-etc.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-uri-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-zero-exit.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l3-container.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l3-hosted.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l3.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-b-l2.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/promoted-c2.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/promotion-nonpromotable.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-calendar-invalid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-non-iso.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-offset-invalid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-submilli.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/same-surface-level-ambiguity.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ssh-id-rsa-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-classification-bare-string.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-classification-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-duplicate-producer-identity.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-empty-source-surface.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-identity-grammar.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-missing-producer-identity.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-missing-run-link-prefix.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-shared-run-link-prefix-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-shared-source-surface.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/two-level-surface-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/unverifiable-probe-evidence.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/vendor-hosted-auto.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/verification-below-floor.json # knowledge youtube-digest skill: this data file lives under evals/fixtures/ but # is named only from SKILL.md / vendor TUNING.md prose — no eval files[] entry From 57e8c6907871b90584e2e393b5ef60ac7b883f9e Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 08:45:21 -0400 Subject: [PATCH 04/12] fix(ci): refresh orphaned-fixture baseline for main-side escalation fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merging origin/main brought in 7 new ungraded security-binding fixtures (escalation-*.json, from the WP runner-escalation lane) that no eval files[] entry or test consumes. The gate correctly red-lined them on the merge ref — it is catching NEW ungraded fixtures exactly as designed — so the exact-path baseline snapshot must be refreshed against the merged tree rather than the point where it was first cut. Add the 7 escalation-* paths to the snapshot (now 181 = 180 autonomy + 1 knowledge). This is debt-tracking, not a grader wiring: the escalation fixtures belong to the same ungraded security-binding class already owned by issue #662, whose cleanup scope now covers them. Verified on the merged tree (da0d5d4): the enumerated set equals a fresh empty-baseline discover (181, identical); --check passes; a probe fixture dropped under security-binding/ is still red-lined (exit 1); the script test suite stays green (7/7); changelog-parity --check passes. Co-Authored-By: Claude Fable 5 --- scripts/orphaned-fixtures-baseline.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/orphaned-fixtures-baseline.txt b/scripts/orphaned-fixtures-baseline.txt index dfe0148fa..0476e6f33 100644 --- a/scripts/orphaned-fixtures-baseline.txt +++ b/scripts/orphaned-fixtures-baseline.txt @@ -30,6 +30,13 @@ plugins/autonomy/skills/setup/evals/fixtures/security-binding/c2-auto-unratified plugins/autonomy/skills/setup/evals/fixtures/security-binding/c3-blocking-unratified.json plugins/autonomy/skills/setup/evals/fixtures/security-binding/c4-auto-merge.json plugins/autonomy/skills/setup/evals/fixtures/security-binding/equal-specificity-tie.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-ack-invalid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-ack-loop.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-push-without-channel.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-runner-route-missing.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-severity-bad-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-severity-tracker-only-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-severity-valid.json plugins/autonomy/skills/setup/evals/fixtures/security-binding/evidence-contrary.json plugins/autonomy/skills/setup/evals/fixtures/security-binding/evidence-stale.json plugins/autonomy/skills/setup/evals/fixtures/security-binding/hosted-ephemeral-l3-valid.json From 0b6f1b1303567880216ae11bbdd5088a81132e0a Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 09:05:57 -0400 Subject: [PATCH 05/12] fix(ci): close two fail-opens in the orphaned-fixture and CHANGELOG gates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both gates could pass wrongly on inputs their own matching logic mishandled. 1. Orphaned-fixture baseline was consumed as a PREFIX, not an exact path. `[[ $path == $entry* ]]` meant a baselined `.../valid.json` also shadowed a new unconsumed `.../valid.json.bak` or `.../valid.jsonl` sibling, so an ungraded fixture could be added next to a grandfathered one and slip the gate. Match by full-string equality (`[[ $path == $entry ]]`) in both is_grandfathered and the staleness scan; rename prefix->entry throughout so the glob cannot be reintroduced. This also finishes closing the round-1 directory-grandfathering hole: an exact path shadows only itself. 2. CHANGELOG-parity --check-bump only checked `git diff --quiet` — that the changelog was touched at all. A whitespace/title/old-entry edit satisfied a version bump. Require the bumped head version's own `## []` entry to exist in the changelog at head (fixed-string grep, tolerant of a trailing date; a missing file counts as undocumented). Strictness is deliberate: a bump documented as `## 1.1.0` (no brackets) fails — enforcing the repo's Keep-a-Changelog `## [x.y.z]` heading format. Comments in both scripts and the orphaned-fixtures baseline header updated to describe exact-match / version-entry semantics (the prior "same idiom as docs-only-paths.txt" prefix framing no longer holds for orphaned fixtures). Tests: orphaned-fixtures gains a `.bak`/`.jsonl` sibling red-lines case and its two prefix-baseline cases become exact-path cases (8/8). changelog-parity gains a bump + unrelated-changelog-edit fails case; the pass case moves to the real `## [x.y.z]` format (12/12). Verified on the merged tree: both self-tests green; orphaned --check and changelog --check pass; the CI-identical --check-bump origin/main passes; a real `.bak` sibling of a baselined fixture now red-lines (exit 1) where the prefix match would have exempted it; shellcheck clean at --severity=style. Co-Authored-By: Claude Fable 5 --- scripts/check-changelog-parity.sh | 26 ++++++++----- scripts/check-changelog-parity.test.sh | 23 ++++++++++-- scripts/check-orphaned-fixtures.sh | 49 +++++++++++++------------ scripts/check-orphaned-fixtures.test.sh | 24 +++++++++--- scripts/orphaned-fixtures-baseline.txt | 12 +++--- 5 files changed, 87 insertions(+), 47 deletions(-) diff --git a/scripts/check-changelog-parity.sh b/scripts/check-changelog-parity.sh index 72aed4f55..fc9d188b1 100755 --- a/scripts/check-changelog-parity.sh +++ b/scripts/check-changelog-parity.sh @@ -7,19 +7,22 @@ # scripts/check-changelog-parity.sh --check-bump fail if a plugin's # manifest version # changed vs but -# its CHANGELOG.md was -# not touched in the diff +# its CHANGELOG.md has no +# entry for the new +# version at head # # Two complementary gaps the same audit surfaced: # * --check is the static repo-wide invariant: a plugins//.claude-plugin/ # plugin.json carrying a `version` must ship a plugins//CHANGELOG.md. # It catches a plugin that has bumped versions but never kept a changelog at # all (autonomy shipped 5 minor bumps with none). -# * --check-bump is the go-forward PR discipline: a version change with no -# CHANGELOG.md edit in the same diff means the release is undocumented. It -# mirrors the sync-*.sh --check-bump bump gates and applies to EVERY plugin, -# grandfathered or not — the moment a debt-listed plugin bumps again it must -# start its changelog. +# * --check-bump is the go-forward PR discipline: a version change whose new +# version has no `## []` entry in the plugin's CHANGELOG.md at head +# means the release is undocumented. Checking for the version's own entry +# (not merely that the file was touched) is deliberate: an unrelated edit — +# whitespace, the title, an old release — must not satisfy the gate. It +# applies to EVERY plugin, grandfathered or not — the moment a debt-listed +# plugin bumps again it must start its changelog. # # Existing "versioned but changelog-less" debt is grandfathered by plugin NAME in # scripts/changelog-parity-baseline.txt (same stale-guarded idiom as @@ -127,8 +130,13 @@ for manifest in "${manifests[@]}"; do head_version="$(version_of "$manifest")" [[ "$head_version" != "$base_version" ]] || continue - if git diff --quiet "$base" -- "$changelog"; then - echo "UNDOCUMENTED BUMP: $name went $base_version -> $head_version but $changelog was not updated in this diff." >&2 + # Require the bumped version's own entry at head, not merely that the file + # changed: an unrelated edit (whitespace, title, an old release) must not + # satisfy the gate. Fixed-string match tolerates a trailing "- " and + # keeps the dotted version and literal brackets literal; a missing file counts + # as undocumented. + if [[ ! -f "$changelog" ]] || ! grep -Fq "## [$head_version]" "$changelog"; then + echo "UNDOCUMENTED BUMP: $name went $base_version -> $head_version but $changelog has no '## [$head_version]' entry at head." >&2 undocumented=$((undocumented + 1)) fi done diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh index bd4333bd1..04af6ebf4 100755 --- a/scripts/check-changelog-parity.test.sh +++ b/scripts/check-changelog-parity.test.sh @@ -79,16 +79,33 @@ rm -rf "$repo" # ============================ --check-bump (diff) ========================= -# version changed AND changelog updated -> passes +# version changed AND changelog has the new version's entry -> passes repo="$(mk_repo)" git_init "$repo" mk_plugin "$repo" alpha 1.0.0 yes git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base base="$(git -C "$repo" rev-parse HEAD)" printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" -printf '# Changelog\n\n## 1.1.0\n' >"$repo/plugins/alpha/CHANGELOG.md" +printf '# Changelog\n\n## [1.1.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump -if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "bump + changelog edit passes --check-bump"; else fail "bump+changelog wrongly failed"; fi +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "bump + '## [x.y.z]' entry passes --check-bump"; else fail "bump+entry wrongly failed"; fi +rm -rf "$repo" + +# SYNTHETIC UNDOCUMENTED BUMP: version changed, changelog edited but WITHOUT an +# entry for the new version (unrelated edit) -> fails. Proves the gate checks +# the version's own entry, not merely that the file was touched. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog (typo fix)\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* ]]; then ok "bump + unrelated changelog edit (no new-version entry) fails --check-bump"; else fail "unrelated-edit bump not caught: rc=$rc out='$out'"; fi rm -rf "$repo" # SYNTHETIC UNDOCUMENTED BUMP: version changed, changelog untouched -> fails diff --git a/scripts/check-orphaned-fixtures.sh b/scripts/check-orphaned-fixtures.sh index 3e8ac172b..cccb8ed15 100755 --- a/scripts/check-orphaned-fixtures.sh +++ b/scripts/check-orphaned-fixtures.sh @@ -10,7 +10,7 @@ # whether a grader consumes it # scripts/check-orphaned-fixtures.sh --check fail if an un-grandfathered # fixture is orphaned, or a -# baseline prefix is now stale +# baseline entry is now stale # # A fixture at /evals/fixtures/ is CONSUMED when any of: # * its skill-relative path (evals/fixtures/) appears in the sibling @@ -31,10 +31,12 @@ # name-matcher cannot honestly grade them. # # Existing orphan debt owned by another issue is grandfathered in -# scripts/orphaned-fixtures-baseline.txt (path-prefix lines, same idiom as -# scripts/docs-only-paths.txt). A prefix there is a promise the owning issue -# burns it down; --check fails on a stale prefix (one that no longer shadows any -# orphan) so the baseline cannot outlive its debt. +# scripts/orphaned-fixtures-baseline.txt (exact fixture paths, one file per +# line — matched by full-string equality, NOT prefix, so a baselined +# `.json` never shadows a new `.json.bak` or `.jsonl` +# sibling). An entry there is a promise the owning issue burns it down; --check +# fails on a stale entry (one that no longer shadows any orphan) so the baseline +# cannot outlive its debt. # # Fail-closed: a fixture with no resolvable grader is an orphan unless # explicitly grandfathered. FIXTURES_BASELINE overrides the baseline path (test @@ -54,16 +56,17 @@ discover | --check) ;; ;; esac -# Active baseline prefixes: strip inline comments and surrounding blank lines. -prefixes=() +# Active baseline entries: strip inline comments and surrounding blank lines. +# Each entry is one exact fixture path, matched by full-string equality below. +entries=() if [[ -f "$BASELINE" ]]; then - mapfile -t prefixes < <(sed -E 's/#.*//; s/^[[:space:]]+//; s/[[:space:]]+$//' "$BASELINE" | grep -v '^$') + mapfile -t entries < <(sed -E 's/#.*//; s/^[[:space:]]+//; s/[[:space:]]+$//' "$BASELINE" | grep -v '^$') fi is_grandfathered() { - local path="$1" prefix - for prefix in "${prefixes[@]}"; do - if [[ "$path" == "$prefix"* ]]; then + local path="$1" entry + for entry in "${entries[@]}"; do + if [[ "$path" == "$entry" ]]; then return 0 fi done @@ -124,13 +127,13 @@ if ((${#fixtures[@]} > 0)); then mapfile -t -d '' fixtures < <(printf '%s\0' "${fixtures[@]}" | sort -z) fi -# Track which baseline prefixes actually shadow an orphan, to flag stale ones. -declare -A prefix_used -matched_prefix() { - local path="$1" prefix - for prefix in "${prefixes[@]}"; do - if [[ "$path" == "$prefix"* ]]; then - printf '%s' "$prefix" +# Track which baseline entries actually shadow an orphan, to flag stale ones. +declare -A entry_used +matched_baseline() { + local path="$1" entry + for entry in "${entries[@]}"; do + if [[ "$path" == "$entry" ]]; then + printf '%s' "$entry" return 0 fi done @@ -156,8 +159,8 @@ for f in "${fixtures[@]}"; do if consumed "$f"; then continue fi - if p="$(matched_prefix "$f")"; then - prefix_used["$p"]=1 + if p="$(matched_baseline "$f")"; then + entry_used["$p"]=1 continue fi echo "ORPHANED FIXTURE: $f is under evals/fixtures/ but no eval case references it and no test asserts on it." >&2 @@ -166,9 +169,9 @@ for f in "${fixtures[@]}"; do done stale=0 -for prefix in "${prefixes[@]}"; do - if [[ -z "${prefix_used[$prefix]:-}" ]]; then - echo "STALE BASELINE: '$prefix' in $BASELINE no longer shadows any orphaned fixture — remove it." >&2 +for entry in "${entries[@]}"; do + if [[ -z "${entry_used[$entry]:-}" ]]; then + echo "STALE BASELINE: '$entry' in $BASELINE no longer shadows any orphaned fixture — remove it." >&2 stale=$((stale + 1)) fi done diff --git a/scripts/check-orphaned-fixtures.test.sh b/scripts/check-orphaned-fixtures.test.sh index fba069157..c9e4662d2 100755 --- a/scripts/check-orphaned-fixtures.test.sh +++ b/scripts/check-orphaned-fixtures.test.sh @@ -69,20 +69,32 @@ rc=$? if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"orphan.md"* ]]; then ok "un-consumed fixture fails --check (synthetic orphan caught)"; else fail "synthetic orphan not caught: rc=$rc out='$out'"; fi rm -rf "$repo" -# --- orphan grandfathered by a baseline prefix -> passes ------------------- -repo="$(mk_repo $'plugins/p/skills/s/evals/fixtures/\n')" +# --- orphan grandfathered by an exact baseline path -> passes -------------- +repo="$(mk_repo $'plugins/p/skills/s/evals/fixtures/orphan.md\n')" seed_skill "$repo" "plugins/p/skills/s" '' printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/orphan.md" -if run_check "$repo" >/dev/null; then ok "grandfathered orphan passes --check"; else fail "grandfathered orphan wrongly failed"; fi +if run_check "$repo" >/dev/null; then ok "grandfathered orphan (exact path) passes --check"; else fail "grandfathered orphan wrongly failed"; fi rm -rf "$repo" -# --- STALE baseline prefix (shadows no orphan) -> fails -------------------- -repo="$(mk_repo $'plugins/p/skills/s/evals/fixtures/\n')" +# --- baseline entry is EXACT, not a prefix: a .bak/.jsonl sibling of a +# baselined file is NOT grandfathered and red-lines -------------------- +repo="$(mk_repo $'plugins/p/skills/s/evals/fixtures/valid.json\n')" +seed_skill "$repo" "plugins/p/skills/s" '' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json" +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json.bak" +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.jsonl" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"valid.json.bak"* && "$out" == *"valid.jsonl"* ]]; then ok "prefix-sibling of a baselined path red-lines (exact-match, no grandfather leak)"; else fail "baseline prefix-leak not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# --- STALE baseline entry (shadows no orphan) -> fails --------------------- +repo="$(mk_repo $'plugins/p/skills/s/evals/fixtures/gone.md\n')" seed_skill "$repo" "plugins/p/skills/s" '"evals/fixtures/used.md"' printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/used.md" out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" rc=$? -if [[ $rc -ne 0 && "$out" == *"STALE BASELINE"* ]]; then ok "stale baseline prefix fails --check"; else fail "stale baseline not caught: rc=$rc out='$out'"; fi +if [[ $rc -ne 0 && "$out" == *"STALE BASELINE"* ]]; then ok "stale baseline entry fails --check"; else fail "stale baseline not caught: rc=$rc out='$out'"; fi rm -rf "$repo" # --- discover mode labels every fixture ------------------------------------ diff --git a/scripts/orphaned-fixtures-baseline.txt b/scripts/orphaned-fixtures-baseline.txt index 0476e6f33..4ade26430 100644 --- a/scripts/orphaned-fixtures-baseline.txt +++ b/scripts/orphaned-fixtures-baseline.txt @@ -1,10 +1,10 @@ # Grandfathered orphaned eval fixtures — existing debt this gate must not -# red-line, each owned by (or awaiting) a burn-down issue. Exact-path lines, -# same matching idiom as scripts/docs-only-paths.txt (a fixture whose path -# starts with an active line is exempt from check-orphaned-fixtures.sh --check), -# but each line pins one file, never a directory: this is a snapshot of today's -# known orphans, so a NEW orphan added under an already-listed directory matches -# no line and is red-lined by the gate rather than silently grandfathered. +# red-line, each owned by (or awaiting) a burn-down issue. Each line is one +# exact fixture path, matched by full-string EQUALITY (not prefix): a line +# exempts only that file from check-orphaned-fixtures.sh --check. This is a +# snapshot of today's known orphans, so a NEW orphan — whether added under an +# already-listed directory or as a `.json.bak` / `.jsonl` sibling +# of a listed file — matches no line and is red-lined rather than grandfathered. # # An entry here is a standing promise the debt is tracked, not a place to hide # new orphans: --check fails on a STALE entry (one that no longer shadows any From c1c3830c173c6365f6bec0681ee71ad454307004 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:38:58 -0400 Subject: [PATCH 06/12] fix(ci): normalize changelogs and close two more --check-bump fail-opens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up hardening on the CHANGELOG-parity gate plus the normalization the gate now requires. 1. Normalize five plugins' CHANGELOG headings to the documented bracketed Keep-a-Changelog form (`## [x.y.z]`), preserving each file's existing date suffix: discovery, docs-hygiene, knowledge, playbooks, session-flow. These bump daily; unbracketed headings would false-fail --check-bump on their next bump as a CHANGELOG FORMAT error. Format-only — no version bumps, no content changes. 2. Split the --check-bump failure taxonomy. A bumped version documented with an unbracketed heading (`## `) is now a distinct CHANGELOG FORMAT error that names the found heading and the expected bracketed form, rather than an UNDOCUMENTED BUMP. An author can fix format in place; conflating it with a missing release entry was misleading. 3. Close the pre-existing-entry fail-open. --check-bump passed whenever the bumped version's `## []` heading was merely present at head — even if it already existed in the base changelog, meaning the bump reused an old heading and shipped no new release note. Require the entry to be ADDED by the change set: present at head AND absent from the changelog at . A reused pre-existing heading is now reported as PRE-EXISTING CHANGELOG ENTRY. 4. Cite issue #688 in the orphaned-fixtures baseline comment for variation-matrix-backlog.json, replacing the un-owned "needs triage" note now that the fixture has a needs-triage grade-or-demote issue. Tests: check-changelog-parity gains a FORMAT-split case, a newly-added-entry pass case, and a pre-existing-entry fail case (15/15). Verified on the merged tree (branch + origin/main): both self-tests green, --check and the CI-identical --check-bump origin/main green; shellcheck clean at --severity=style. Co-Authored-By: Claude Fable 5 --- plugins/discovery/CHANGELOG.md | 14 +++--- plugins/docs-hygiene/CHANGELOG.md | 10 ++--- plugins/knowledge/CHANGELOG.md | 32 +++++++------- plugins/playbooks/CHANGELOG.md | 4 +- plugins/session-flow/CHANGELOG.md | 24 +++++----- scripts/check-changelog-parity.sh | 61 +++++++++++++++++++------- scripts/check-changelog-parity.test.sh | 47 ++++++++++++++++++++ scripts/orphaned-fixtures-baseline.txt | 6 +-- 8 files changed, 138 insertions(+), 60 deletions(-) diff --git a/plugins/discovery/CHANGELOG.md b/plugins/discovery/CHANGELOG.md index 72ddaf591..3d304e738 100644 --- a/plugins/discovery/CHANGELOG.md +++ b/plugins/discovery/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog — discovery plugin -## 0.7.3 — 2026-07-19 +## [0.7.3] — 2026-07-19 ### Fixed @@ -11,7 +11,7 @@ current working directory when no repo root exists. The live root stays available for resolving files while working; it is never echoed into the handoff, so `EXPLORE.md` stays machine-agnostic. -## 0.7.2 — 2026-07-19 +## [0.7.2] — 2026-07-19 ### Changed @@ -21,7 +21,7 @@ write path — replacing the dead `docs/adr/…` reference that resolves to nothing in the cache-isolated installed plugin. Behavior is unchanged; gitbook stays deferred and non-writable. -## 0.7.0 — 2026-07-18 +## [0.7.0] — 2026-07-18 ### Changed @@ -34,7 +34,7 @@ interview, so headless and CI use are possible. The ignore guard, the gitbook-deferred handling, and the never-edit-root-`.gitignore` rule are unchanged. -## 0.6.0 — 2026-07-17 +## [0.6.0] — 2026-07-17 ### Changed @@ -45,7 +45,7 @@ artifacts there directly; only workers dispatched into their own checkout return findings by value for the parent to write. -## 0.5.1 — 2026-07-15 +## [0.5.1] — 2026-07-15 ### Fixed @@ -54,7 +54,7 @@ and states that durable writes still target `docs`; the skill never configures or tests a GitBook API, MCP, or Git Sync writer. -## 0.5.0 — 2026-07-15 +## [0.5.0] — 2026-07-15 ### Added @@ -65,7 +65,7 @@ doubled phase/query/source minimums (SKILL.md item 8, discipline.md's "Broad-topic auto-detect"). -## 0.4.0 — 2026-07-14 +## [0.4.0] — 2026-07-14 Adopt the marketplace topic-docs convention, contract v1.0.0 (): diff --git a/plugins/docs-hygiene/CHANGELOG.md b/plugins/docs-hygiene/CHANGELOG.md index dbf68503d..29b1894ea 100644 --- a/plugins/docs-hygiene/CHANGELOG.md +++ b/plugins/docs-hygiene/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog — docs-hygiene plugin -## 0.7.0 — 2026-07-18 +## [0.7.0] — 2026-07-18 Changed: @@ -12,7 +12,7 @@ Changed: ambient, Git Bash on native Windows), the compress-only `markdownlint-cli2` requirement with its absence behavior, and the optional `caveman` backend. -## 0.6.0 — 2026-07-17 +## [0.6.0] — 2026-07-17 Changed: @@ -24,7 +24,7 @@ Changed: word. The detect-script env vars moved with it: `DECLUTTER_REPO_ROOT` → `AUDIT_NOISE_REPO_ROOT`. -## 0.5.0 — 2026-07-15 +## [0.5.0] — 2026-07-15 Changed: @@ -32,7 +32,7 @@ Changed: `/docs-hygiene:encapsulation-audit` invocations to `/docs-hygiene:audit-encapsulation`; the plugin ID (`docs-hygiene`) is unchanged, only the skill's leaf name moved. -## 0.4.0 — 2026-07-15 +## [0.4.0] — 2026-07-15 Added: @@ -44,7 +44,7 @@ Added: - The "add an eval case" clause, re-added to the two Gotchas/Recheck-trigger bullets in rename-references/SKILL.md. -## 0.3.0 — 2026-07-14 +## [0.3.0] — 2026-07-14 Adopt the marketplace topic-docs convention (`docs/conventions/topic-docs/`, contract v1.0.0) in the declutter diff --git a/plugins/knowledge/CHANGELOG.md b/plugins/knowledge/CHANGELOG.md index 10f35f91c..5ff9f1458 100644 --- a/plugins/knowledge/CHANGELOG.md +++ b/plugins/knowledge/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to the `knowledge` plugin are recorded here. The `version` i `.claude-plugin/plugin.json` is the delivery vehicle — a consumer receives a change only after that version increases. -## 0.8.0 +## [0.8.0] ### Changed @@ -18,7 +18,7 @@ only after that version increases. scripts the ingest skills already run — pulling the prerequisite/provisioning surface onto the setup contract. The personal env-channel scalars are unchanged. -## 0.7.1 +## [0.7.1] ### Changed @@ -27,7 +27,7 @@ only after that version increases. the EPUB branch's `unzip` requirement (not bundled with Git Bash) — cross-platform declaration wave. PDF-only use needs neither extra install. -## 0.7.0 +## [0.7.0] ### Changed @@ -54,13 +54,13 @@ only after that version increases. plaintext in `~/.claude/.credentials.json` on Windows, so those secrets stay in shell env until keychain-backed sensitive storage lands there (documented at the course-digest auth-env site). -## 0.6.1 +## [0.6.1] ### Changed - References to the renamed `/planning:plan` skill (was `/planning:architect`, planning 0.13.0 breaking rename) retargeted. Version bumped so existing installs receive the rewritten prompts. -## 0.6.0 +## [0.6.0] ### Changed @@ -72,7 +72,7 @@ only after that version increases. unaffected (`.work//...` layout is unchanged); resume with `/knowledge:youtube-digest resume `. -## 0.5.6 +## [0.5.6] ### Added @@ -80,7 +80,7 @@ only after that version increases. graduate into a standalone plugin (once its vendored `video-digestion` package is independently distributable). -## 0.5.5 +## [0.5.5] ### Fixed @@ -88,7 +88,7 @@ only after that version increases. linear pass, but now preserves an unmatched literal less-than tail instead of truncating the rest of programming, mathematics, or other tolerant transcript text. -## 0.5.4 +## [0.5.4] ### Fixed @@ -101,7 +101,7 @@ only after that version increases. clone option parsing is terminated explicitly; and WebVTT/entity cleanup uses bounded, single-pass transformations that cannot turn nested malformed input into active markup. -## 0.5.3 +## [0.5.3] ### Changed @@ -115,7 +115,7 @@ only after that version increases. now records that the `verification/` lane name matches the convention's canon. Docs-only — no paths or behavior change; the `library_dir` seam is untouched. -## 0.5.2 +## [0.5.2] ### Fixed @@ -141,7 +141,7 @@ only after that version increases. forces `--no-playlist`; and a passing key-frame quality-audit row now requires a substantive evidence note (an omitted note no longer bypasses the gate). -## 0.5.1 +## [0.5.1] ### Changed @@ -153,7 +153,7 @@ only after that version increases. into `${CLAUDE_PLUGIN_DATA}` is unchanged. Internal restructure — no consumer-facing behavior change; the version bump delivers the moved source (and the new install fingerprint) to consumers. -## 0.5.0 +## [0.5.0] ### Added @@ -189,7 +189,7 @@ only after that version increases. - `repo-analysis` and `video-digestion` are now vendored by both the youtube and course-digest skills. Deduplication of the two copies is tracked separately. -## 0.4.0 +## [0.4.0] ### Changed @@ -208,7 +208,7 @@ only after that version increases. land, restoring the stronger wording softened while the seam was unwired (`book-distill` remains the documented exception). -## 0.3.0 +## [0.3.0] ### Changed @@ -216,7 +216,7 @@ only after that version increases. hardening so synthesized artifacts land at the configured library directory in the consuming repo. -## 0.2.0 +## [0.2.0] ### Added @@ -237,6 +237,6 @@ only after that version increases. dependencies resolve from `${CLAUDE_PLUGIN_DATA}` (NODE_PATH is CommonJS-only and does not apply). -## 0.1.1 +## [0.1.1] - Prior baseline: `book-distill` and `setup` skills. diff --git a/plugins/playbooks/CHANGELOG.md b/plugins/playbooks/CHANGELOG.md index 9ea486e09..f2f3623d4 100644 --- a/plugins/playbooks/CHANGELOG.md +++ b/plugins/playbooks/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to the `playbooks` plugin are recorded here. The `version` i `.claude-plugin/plugin.json` is the delivery vehicle — a consumer receives a change only after that version increases. -## 0.2.0 +## [0.2.0] ### Changed @@ -17,7 +17,7 @@ only after that version increases. drift-check mechanics now point at the renamed pack path. Only the wrapper skill name (directory, frontmatter `name`, and references) changed. -## 0.1.0 +## [0.1.0] ### Added diff --git a/plugins/session-flow/CHANGELOG.md b/plugins/session-flow/CHANGELOG.md index e6c4bdab8..1e02c5a1a 100644 --- a/plugins/session-flow/CHANGELOG.md +++ b/plugins/session-flow/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog — session-flow plugin -## 0.10.2 — 2026-07-19 +## [0.10.2] — 2026-07-19 Fixed: @@ -10,7 +10,7 @@ Fixed: copy-as-is run of the snippet previously bypassed the memory_dir seam, missing the handoff chain in any repo that relocates its memory tier. -## 0.10.1 — 2026-07-19 +## [0.10.1] — 2026-07-19 Changed: @@ -20,7 +20,7 @@ Changed: own sections and keeps only the plugin-specific no-project-root fallback detail. -## 0.10.0 — 2026-07-18 +## [0.10.0] — 2026-07-18 Added: @@ -38,7 +38,7 @@ Added: worktree mechanics route to whatever capabilities are installed, falling back to direct git / gh. The plugin now bundles seven skills. -## 0.9.1 — 2026-07-18 +## [0.9.1] — 2026-07-18 Fixed: @@ -50,7 +50,7 @@ Fixed: model defaults to `inherit` per the subagents doc (resolution order and cost-control quote now cited in `context/sources.md`). -## 0.9.0 — 2026-07-18 +## [0.9.0] — 2026-07-18 Added: @@ -63,7 +63,7 @@ Added: re-anchors; it does not resume the work (the keep-going sibling), enumerate worktrees, or triage PR feedback. The plugin now bundles six skills. -## 0.8.0 — 2026-07-17 +## [0.8.0] — 2026-07-17 Changed: @@ -72,13 +72,13 @@ Changed: the contract's `.worktreeinclude` template carries into new worktrees, while handoffs are session-scoped and deliberately not carried. -## 0.7.1 +## [0.7.1] ### Changed - References to the renamed `/planning:plan` skill (was `/planning:architect`, planning 0.13.0 breaking rename) retargeted. Version bumped so existing installs receive the rewritten prompts. -## 0.7.0 — 2026-07-17 +## [0.7.0] — 2026-07-17 Added: @@ -93,7 +93,7 @@ Added: the interruption cause is deliberately not diagnosed (recovery is identical regardless). The plugin now bundles five skills. -## 0.6.0 — 2026-07-16 +## [0.6.0] — 2026-07-16 Changed: @@ -113,7 +113,7 @@ Added: pressure, and concurrent-session / rate-limit headroom, with small/medium/large fan-out sizing and single-agent as the floor. -## 0.5.0 — 2026-07-15 +## [0.5.0] — 2026-07-15 Added: @@ -145,7 +145,7 @@ Added: actual capability inventory whenever capabilities are added, renamed, or retired. -## 0.4.0 — 2026-07-15 +## [0.4.0] — 2026-07-15 Added: @@ -159,7 +159,7 @@ Added: handoff without verifiable sanity-check evidence, continuing past an explicit stop). Loaded on demand from the SKILL.md checklist. -## 0.3.0 — 2026-07-14 +## [0.3.0] — 2026-07-14 Adopt the marketplace topic-docs convention (`docs/conventions/topic-docs/`, contract v1.0.0): diff --git a/scripts/check-changelog-parity.sh b/scripts/check-changelog-parity.sh index fc9d188b1..09937351f 100755 --- a/scripts/check-changelog-parity.sh +++ b/scripts/check-changelog-parity.sh @@ -7,22 +7,29 @@ # scripts/check-changelog-parity.sh --check-bump fail if a plugin's # manifest version # changed vs but -# its CHANGELOG.md has no +# its CHANGELOG.md does +# not ADD a `## []` # entry for the new -# version at head +# version (present at +# head, absent at ) # # Two complementary gaps the same audit surfaced: # * --check is the static repo-wide invariant: a plugins//.claude-plugin/ # plugin.json carrying a `version` must ship a plugins//CHANGELOG.md. # It catches a plugin that has bumped versions but never kept a changelog at # all (autonomy shipped 5 minor bumps with none). -# * --check-bump is the go-forward PR discipline: a version change whose new -# version has no `## []` entry in the plugin's CHANGELOG.md at head -# means the release is undocumented. Checking for the version's own entry -# (not merely that the file was touched) is deliberate: an unrelated edit — -# whitespace, the title, an old release — must not satisfy the gate. It -# applies to EVERY plugin, grandfathered or not — the moment a debt-listed -# plugin bumps again it must start its changelog. +# * --check-bump is the go-forward PR discipline: a version change must ADD a +# `## []` entry for the new version — present in the plugin's +# CHANGELOG.md at head AND absent from it at . Two failure classes the +# gate must not conflate: (a) checking for the version's own entry, not +# merely that the file was touched, so an unrelated edit — whitespace, the +# title, an old release — cannot satisfy the gate; (b) requiring the entry +# to be newly added, so reusing a heading that already existed at +# (a bump with no fresh release note) cannot satisfy it either. An entry +# documented as `## ` (no brackets) is a separate FORMAT failure, +# reported as such rather than as undocumented. It applies to EVERY plugin, +# grandfathered or not — the moment a debt-listed plugin bumps again it must +# start its changelog. # # Existing "versioned but changelog-less" debt is grandfathered by plugin NAME in # scripts/changelog-parity-baseline.txt (same stale-guarded idiom as @@ -118,6 +125,8 @@ if ! git rev-parse --verify --quiet "${base}^{commit}" >/dev/null; then fi undocumented=0 +malformed=0 +preexisting=0 for manifest in "${manifests[@]}"; do plugin_dir="${manifest%/.claude-plugin/plugin.json}" name="${plugin_dir##*/}" @@ -133,16 +142,38 @@ for manifest in "${manifests[@]}"; do # Require the bumped version's own entry at head, not merely that the file # changed: an unrelated edit (whitespace, title, an old release) must not # satisfy the gate. Fixed-string match tolerates a trailing "- " and - # keeps the dotted version and literal brackets literal; a missing file counts - # as undocumented. - if [[ ! -f "$changelog" ]] || ! grep -Fq "## [$head_version]" "$changelog"; then + # keeps the dotted version and literal brackets literal. + if [[ -f "$changelog" ]] && grep -Fq "## [$head_version]" "$changelog"; then + # The release entry must be ADDED by this change set, not merely present: + # a heading that already existed in the changelog at $base means the bump + # reused a pre-existing entry and shipped no new release note. Require it + # absent from the base changelog. (git show fails for a changelog that is + # new at $base -> empty -> counts as absent, which is correct: it was added + # here.) + if git show "$base:$changelog" 2>/dev/null | grep -Fq "## [$head_version]"; then + echo "PRE-EXISTING CHANGELOG ENTRY: $name bumped $base_version -> $head_version but '## [$head_version]' already existed in $changelog at $base; add the release entry in this change set." >&2 + preexisting=$((preexisting + 1)) + fi + continue + fi + + # Split the failure: a heading that names the version but omits the brackets + # (## ) is a FORMAT error the author can fix in place, not a missing + # release. Escape dots so the version matches literally. + esc="${head_version//./\\.}" + if found="$([[ -f "$changelog" ]] && grep -m1 -E "^##[[:space:]]+${esc}([[:space:]]|\$)" "$changelog")"; then + echo "CHANGELOG FORMAT: $name $head_version is documented as '${found}' but must use the bracketed Keep-a-Changelog heading '## [$head_version]'." >&2 + malformed=$((malformed + 1)) + else echo "UNDOCUMENTED BUMP: $name went $base_version -> $head_version but $changelog has no '## [$head_version]' entry at head." >&2 undocumented=$((undocumented + 1)) fi done -if ((undocumented > 0)); then - echo "Add a CHANGELOG.md entry for every plugin whose version changed." >&2 +if ((undocumented > 0 || malformed > 0 || preexisting > 0)); then + ((undocumented > 0)) && echo "Add a '## []' entry for every plugin whose version changed." >&2 + ((malformed > 0)) && echo "Convert unbracketed changelog headings to the '## []' Keep-a-Changelog form." >&2 + ((preexisting > 0)) && echo "Add the bumped version's '## []' entry in this change set; it must be absent from the base changelog, not merely present at head." >&2 exit 1 fi -echo "Every plugin whose version changed vs $base also updated its CHANGELOG.md." +echo "Every plugin whose version changed vs $base has a '## []' CHANGELOG.md entry." diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh index 04af6ebf4..1cd51f75c 100755 --- a/scripts/check-changelog-parity.test.sh +++ b/scripts/check-changelog-parity.test.sh @@ -91,6 +91,53 @@ git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "bump + '## [x.y.z]' entry passes --check-bump"; else fail "bump+entry wrongly failed"; fi rm -rf "$repo" +# SYNTHETIC MALFORMED ENTRY: version present but as an UNBRACKETED heading +# (## 1.1.0) -> FORMAT error naming the found heading, NOT "UNDOCUMENTED BUMP". +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n## 1.1.0 — 2026-07-20\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"CHANGELOG FORMAT"*"alpha"* && "$out" == *"## 1.1.0"* && "$out" != *"UNDOCUMENTED BUMP"* ]]; then ok "bump + unbracketed heading -> FORMAT error (not UNDOCUMENTED)"; else fail "format-split not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# NEWLY-ADDED ENTRY: base carries an earlier `## [1.0.0]`; the bump ADDS +# `## [1.1.0]` (present at head, absent at base) -> passes. Proves the pass +# path accepts a genuinely new release note. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n## [1.1.0]\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "bump adding a NEW '## [x.y.z]' entry (absent at base) passes --check-bump"; else fail "newly-added entry wrongly failed"; fi +rm -rf "$repo" + +# PRE-EXISTING ENTRY: the `## [1.1.0]` heading already exists in the base +# changelog; the bump only edits plugin.json, adding no new release note -> +# fails as PRE-EXISTING (not UNDOCUMENTED). Proves the gate closes the fail-open +# where a bump reuses a heading that predates the change set. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.1.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"PRE-EXISTING CHANGELOG ENTRY"*"alpha"* && "$out" != *"UNDOCUMENTED BUMP"* ]]; then ok "bump reusing a base-pre-existing '## [x.y.z]' entry fails --check-bump"; else fail "preexisting-entry not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + # SYNTHETIC UNDOCUMENTED BUMP: version changed, changelog edited but WITHOUT an # entry for the new version (unrelated edit) -> fails. Proves the gate checks # the version's own entry, not merely that the file was touched. diff --git a/scripts/orphaned-fixtures-baseline.txt b/scripts/orphaned-fixtures-baseline.txt index 4ade26430..cb8b38bad 100644 --- a/scripts/orphaned-fixtures-baseline.txt +++ b/scripts/orphaned-fixtures-baseline.txt @@ -202,7 +202,7 @@ plugins/autonomy/skills/setup/evals/fixtures/security-binding/verification-below # knowledge youtube-digest skill: this data file lives under evals/fixtures/ but # is named only from SKILL.md / vendor TUNING.md prose — no eval files[] entry -# and no test consume it. Surfaced by this gate's initial discover run; it has -# no dedicated owning issue yet and needs triage (wire it into a grader, or move -# it out of evals/fixtures/ if it is reference data rather than a fixture). +# and no test consume it. Surfaced by this gate's initial discover run; tracked +# by issue #688 (wire it into a grader, or move it out of evals/fixtures/ if it +# is reference data rather than a fixture). plugins/knowledge/skills/youtube-digest/evals/fixtures/variation-matrix-backlog.json From 8c028dcebf40a3f157dc879556be712959c54700 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 10:53:32 -0400 Subject: [PATCH 07/12] fix(ci): anchor the changelog release-entry match to a line-start heading Codex P2 on #681: the fixed-string version match also accepted the bumped version appearing in prose or an indented/fenced example, satisfying (or falsely pre-existing) the release entry without a real heading. Both the head and base checks now match an anchored Markdown heading; self-test case added proving a prose/indented mention fails as UNDOCUMENTED. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FM1RfM3jHkgenpdbMv4o64 --- scripts/check-changelog-parity.sh | 14 ++++++++------ scripts/check-changelog-parity.test.sh | 19 +++++++++++++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/scripts/check-changelog-parity.sh b/scripts/check-changelog-parity.sh index 09937351f..60fc43a11 100755 --- a/scripts/check-changelog-parity.sh +++ b/scripts/check-changelog-parity.sh @@ -141,16 +141,19 @@ for manifest in "${manifests[@]}"; do # Require the bumped version's own entry at head, not merely that the file # changed: an unrelated edit (whitespace, title, an old release) must not - # satisfy the gate. Fixed-string match tolerates a trailing "- " and - # keeps the dotted version and literal brackets literal. - if [[ -f "$changelog" ]] && grep -Fq "## [$head_version]" "$changelog"; then + # satisfy the gate. The match is ANCHORED to a line-start Markdown heading + # (^## []) so the version string appearing in prose or a fenced code + # example never satisfies — or falsely pre-exists — the release entry. + esc="${head_version//./\\.}" + heading_re="^## \[${esc}\]" + if [[ -f "$changelog" ]] && grep -Eq "$heading_re" "$changelog"; then # The release entry must be ADDED by this change set, not merely present: # a heading that already existed in the changelog at $base means the bump # reused a pre-existing entry and shipped no new release note. Require it # absent from the base changelog. (git show fails for a changelog that is # new at $base -> empty -> counts as absent, which is correct: it was added # here.) - if git show "$base:$changelog" 2>/dev/null | grep -Fq "## [$head_version]"; then + if git show "$base:$changelog" 2>/dev/null | grep -Eq "$heading_re"; then echo "PRE-EXISTING CHANGELOG ENTRY: $name bumped $base_version -> $head_version but '## [$head_version]' already existed in $changelog at $base; add the release entry in this change set." >&2 preexisting=$((preexisting + 1)) fi @@ -159,8 +162,7 @@ for manifest in "${manifests[@]}"; do # Split the failure: a heading that names the version but omits the brackets # (## ) is a FORMAT error the author can fix in place, not a missing - # release. Escape dots so the version matches literally. - esc="${head_version//./\\.}" + # release. (esc computed above; dots already escaped.) if found="$([[ -f "$changelog" ]] && grep -m1 -E "^##[[:space:]]+${esc}([[:space:]]|\$)" "$changelog")"; then echo "CHANGELOG FORMAT: $name $head_version is documented as '${found}' but must use the bracketed Keep-a-Changelog heading '## [$head_version]'." >&2 malformed=$((malformed + 1)) diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh index 1cd51f75c..90e14fa90 100755 --- a/scripts/check-changelog-parity.test.sh +++ b/scripts/check-changelog-parity.test.sh @@ -138,6 +138,25 @@ rc=$? if [[ $rc -ne 0 && "$out" == *"PRE-EXISTING CHANGELOG ENTRY"*"alpha"* && "$out" != *"UNDOCUMENTED BUMP"* ]]; then ok "bump reusing a base-pre-existing '## [x.y.z]' entry fails --check-bump"; else fail "preexisting-entry not caught: rc=$rc out='$out'"; fi rm -rf "$repo" +# PROSE / CODE-EXAMPLE MENTION: the bumped version's `## [1.1.0]` appears only +# inside prose (an indented example line), never as a line-start heading -> +# fails as UNDOCUMENTED. Proves the heading match is anchored, so a mention in +# body text or a fenced example neither satisfies nor falsely pre-exists the +# release entry. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n## [1.0.0]\n\nNext release will be titled "## [1.1.0]" per convention.\n ## [1.1.0] (example, indented, not a heading)\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "version string in prose/indented example does not satisfy the anchored heading match"; else fail "unanchored-mention not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + # SYNTHETIC UNDOCUMENTED BUMP: version changed, changelog edited but WITHOUT an # entry for the new version (unrelated edit) -> fails. Proves the gate checks # the version's own entry, not merely that the file was touched. From 91571eb1d75ed9244cd55b597a16f61598e931ce Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:02:28 -0400 Subject: [PATCH 08/12] fix(ci): bound fixture-basename matches and drop regex from the heading check Two Codex P2s on #681, one per gate: - orphaned-fixture: grep -w treated "." as a word boundary, so a referenced valid.json also consumed a new unconsumed valid.json.bak sibling. Basename matches now require non-filename-character neighbors around an ERE-escaped basename. - changelog-parity: the heading match interpolated the version into an ERE with only dots escaped, so SemVer build metadata (1.0.1+build.1) broke the pattern and false-failed a documented bump. The heading check is now a fixed-string line-anchored awk match (no regex at all); the FORMAT-split probe escapes all ERE metacharacters. Self-tests: suffix-sibling orphan case + build-metadata round-trip case added; orphan suite 9/9, parity suite 17/17. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FM1RfM3jHkgenpdbMv4o64 --- scripts/check-changelog-parity.sh | 20 ++++++++++++-------- scripts/check-changelog-parity.test.sh | 15 +++++++++++++++ scripts/check-orphaned-fixtures.sh | 13 +++++++++++-- scripts/check-orphaned-fixtures.test.sh | 12 ++++++++++++ 4 files changed, 50 insertions(+), 10 deletions(-) diff --git a/scripts/check-changelog-parity.sh b/scripts/check-changelog-parity.sh index 60fc43a11..d82da3059 100755 --- a/scripts/check-changelog-parity.sh +++ b/scripts/check-changelog-parity.sh @@ -141,19 +141,20 @@ for manifest in "${manifests[@]}"; do # Require the bumped version's own entry at head, not merely that the file # changed: an unrelated edit (whitespace, title, an old release) must not - # satisfy the gate. The match is ANCHORED to a line-start Markdown heading - # (^## []) so the version string appearing in prose or a fenced code - # example never satisfies — or falsely pre-exists — the release entry. - esc="${head_version//./\\.}" - heading_re="^## \[${esc}\]" - if [[ -f "$changelog" ]] && grep -Eq "$heading_re" "$changelog"; then + # satisfy the gate. The match is a FIXED-STRING heading anchored to line + # start (awk index()==1), so the version string appearing in prose or a + # fenced example never satisfies — or falsely pre-exists — the release + # entry, and SemVer metacharacters (1.0.1+build.1) never leak into a regex. + heading="## [${head_version}]" + has_heading() { awk -v h="$heading" 'index($0, h) == 1 { found=1; exit } END { exit !found }'; } + if [[ -f "$changelog" ]] && has_heading <"$changelog"; then # The release entry must be ADDED by this change set, not merely present: # a heading that already existed in the changelog at $base means the bump # reused a pre-existing entry and shipped no new release note. Require it # absent from the base changelog. (git show fails for a changelog that is # new at $base -> empty -> counts as absent, which is correct: it was added # here.) - if git show "$base:$changelog" 2>/dev/null | grep -Eq "$heading_re"; then + if git show "$base:$changelog" 2>/dev/null | has_heading; then echo "PRE-EXISTING CHANGELOG ENTRY: $name bumped $base_version -> $head_version but '## [$head_version]' already existed in $changelog at $base; add the release entry in this change set." >&2 preexisting=$((preexisting + 1)) fi @@ -162,7 +163,10 @@ for manifest in "${manifests[@]}"; do # Split the failure: a heading that names the version but omits the brackets # (## ) is a FORMAT error the author can fix in place, not a missing - # release. (esc computed above; dots already escaped.) + # release. Escape ALL ERE metacharacters (SemVer allows + in build metadata), + # not just dots. + # shellcheck disable=SC2016 # single quotes are deliberate: $ is an ERE metachar being escaped, not a shell expansion + esc="$(printf '%s' "$head_version" | sed -E 's/[][\\.|$(){}?+*^]/\\&/g')" if found="$([[ -f "$changelog" ]] && grep -m1 -E "^##[[:space:]]+${esc}([[:space:]]|\$)" "$changelog")"; then echo "CHANGELOG FORMAT: $name $head_version is documented as '${found}' but must use the bracketed Keep-a-Changelog heading '## [$head_version]'." >&2 malformed=$((malformed + 1)) diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh index 90e14fa90..daf0929d1 100755 --- a/scripts/check-changelog-parity.test.sh +++ b/scripts/check-changelog-parity.test.sh @@ -138,6 +138,21 @@ rc=$? if [[ $rc -ne 0 && "$out" == *"PRE-EXISTING CHANGELOG ENTRY"*"alpha"* && "$out" != *"UNDOCUMENTED BUMP"* ]]; then ok "bump reusing a base-pre-existing '## [x.y.z]' entry fails --check-bump"; else fail "preexisting-entry not caught: rc=$rc out='$out'"; fi rm -rf "$repo" +# SEMVER BUILD METADATA: a version like 1.0.1+build.1 must round-trip — the +# fixed-string heading match must not interpret "+" as a regex quantifier, so +# a correctly-added `## [1.0.1+build.1]` entry passes the gate. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.0.1+build.1" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n## [1.0.1+build.1]\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "SemVer build-metadata version with a proper entry passes (no regex leak)"; else fail "build-metadata version wrongly failed"; fi +rm -rf "$repo" + # PROSE / CODE-EXAMPLE MENTION: the bumped version's `## [1.1.0]` appears only # inside prose (an indented example line), never as a line-start heading -> # fails as UNDOCUMENTED. Proves the heading match is anchored, so a mention in diff --git a/scripts/check-orphaned-fixtures.sh b/scripts/check-orphaned-fixtures.sh index cccb8ed15..55965ce5d 100755 --- a/scripts/check-orphaned-fixtures.sh +++ b/scripts/check-orphaned-fixtures.sh @@ -96,18 +96,27 @@ consumed() { rel="${fixture#"$skill_dir"/}" # evals/fixtures/ base="${fixture##*/}" + # Basename matches are bounded by NON-FILENAME characters on both sides: + # grep -w treats "." as a word boundary, so a referenced valid.json would + # wrongly consume a new unconsumed valid.json.bak sibling. Escape ERE + # metacharacters in the basename, then require the neighbors (if any) to be + # outside [A-Za-z0-9._-] so a longer filename never satisfies the match. + # shellcheck disable=SC2016 # single quotes are deliberate: $ is an ERE metachar being escaped, not a shell expansion + esc_base="$(printf '%s' "$base" | sed -E 's/[][\\.|$(){}?+*^]/\\&/g')" + base_re="(^|[^A-Za-z0-9._-])${esc_base}([^A-Za-z0-9._-]|$)" + if [[ -f "$evals_json" ]]; then if grep -qF "$rel" "$evals_json"; then return 0 fi - if grep -qwF "$base" "$evals_json"; then + if grep -qE "$base_re" "$evals_json"; then return 0 fi fi plugin="$(plugin_root_of "$fixture")" while IFS= read -r -d '' test_file; do - if grep -qwF "$base" "$test_file"; then + if grep -qE "$base_re" "$test_file"; then return 0 fi done < <(find "$plugin" -type f -name '*.test.*' -print0 2>/dev/null) diff --git a/scripts/check-orphaned-fixtures.test.sh b/scripts/check-orphaned-fixtures.test.sh index c9e4662d2..1a31c6486 100755 --- a/scripts/check-orphaned-fixtures.test.sh +++ b/scripts/check-orphaned-fixtures.test.sh @@ -60,6 +60,18 @@ printf 'assert_on fixtures/by-test.md\n' >"$repo/plugins/p/skills/s/scripts/thin if run_check "$repo" >/dev/null; then ok "test-asserted fixture passes --check"; else fail "test-asserted fixture wrongly flagged"; fi rm -rf "$repo" +# --- REFERENCE-NAME SUFFIX SIBLING: a referenced valid.json must NOT consume +# a new valid.json.bak (grep -w treated "." as a word boundary; the bounded +# match rejects filename-character neighbors) -> .bak sibling is an orphan --- +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '"evals/fixtures/valid.json"' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json" +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json.bak" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"valid.json.bak"* && "$out" != *"ORPHANED FIXTURE"*"fixtures/valid.json is"* ]]; then ok "suffix sibling of a referenced fixture red-lines (bounded basename match)"; else fail "reference-name suffix sibling not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + # --- SYNTHETIC ORPHAN: consumed by nothing -> --check fails ---------------- repo="$(mk_repo)" seed_skill "$repo" "plugins/p/skills/s" '' From edf355b917e02b54c73b7338c876cc4ff8c2cdc6 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:27:31 -0400 Subject: [PATCH 09/12] =?UTF-8?q?fix(ci):=20close=20two=20more=20gate=20fa?= =?UTF-8?q?il-opens=20=E2=80=94=20files[]=20substring=20and=20fenced=20hea?= =?UTF-8?q?dings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two Codex P2s on #681, both fail-open in the gates' matching: 1. check-orphaned-fixtures.sh: the files[] PATH check used an unbounded fixed-string search of the whole evals.json, so an eval referencing evals/fixtures/valid.json.bak marked a NEW unconsumed evals/fixtures/valid.json sibling consumed (shorter $rel is a substring of the longer value). Compare $rel against each JSON string value EXACTLY with jq; the bounded-basename check remains the fallback. (Complements the earlier basename-boundary fix, which left this path branch open.) 2. check-changelog-parity.sh: the awk release-heading predicate matched `## []` at column zero without tracking fenced-code state, so a bump adding only a ``` fenced example containing the heading satisfied the gate — an undocumented release. The predicate now toggles fence state on ``` / ~~~ lines and matches only outside fences; the overclaiming comment is corrected. Self-tests: orphaned adds a files[]-substring case (10/10); changelog adds a fenced-block case (18/18). shellcheck clean at --severity=style and against .shellcheckrc; orphaned --check + set-equality (181, unchanged — the stricter path match reclassified no real consumption), changelog --check, and --check-bump origin/main all green. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01FM1RfM3jHkgenpdbMv4o64 --- scripts/check-changelog-parity.sh | 15 +++++++++---- scripts/check-changelog-parity.test.sh | 28 ++++++++++++++++++++----- scripts/check-orphaned-fixtures.sh | 7 ++++++- scripts/check-orphaned-fixtures.test.sh | 13 ++++++++++++ 4 files changed, 53 insertions(+), 10 deletions(-) diff --git a/scripts/check-changelog-parity.sh b/scripts/check-changelog-parity.sh index d82da3059..2832829b3 100755 --- a/scripts/check-changelog-parity.sh +++ b/scripts/check-changelog-parity.sh @@ -142,11 +142,18 @@ for manifest in "${manifests[@]}"; do # Require the bumped version's own entry at head, not merely that the file # changed: an unrelated edit (whitespace, title, an old release) must not # satisfy the gate. The match is a FIXED-STRING heading anchored to line - # start (awk index()==1), so the version string appearing in prose or a - # fenced example never satisfies — or falsely pre-exists — the release - # entry, and SemVer metacharacters (1.0.1+build.1) never leak into a regex. + # start (awk index()==1) and OUTSIDE fenced code, so the version string + # appearing in prose, an indented line, or a ``` / ~~~ example never + # satisfies — or falsely pre-exists — the release entry, and SemVer + # metacharacters (1.0.1+build.1) never leak into a regex. heading="## [${head_version}]" - has_heading() { awk -v h="$heading" 'index($0, h) == 1 { found=1; exit } END { exit !found }'; } + has_heading() { + awk -v h="$heading" ' + /^[[:space:]]*(```|~~~)/ { infence = !infence; next } + !infence && index($0, h) == 1 { found = 1; exit } + END { exit !found } + ' + } if [[ -f "$changelog" ]] && has_heading <"$changelog"; then # The release entry must be ADDED by this change set, not merely present: # a heading that already existed in the changelog at $base means the bump diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh index daf0929d1..84421cc40 100755 --- a/scripts/check-changelog-parity.test.sh +++ b/scripts/check-changelog-parity.test.sh @@ -153,11 +153,11 @@ git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "SemVer build-metadata version with a proper entry passes (no regex leak)"; else fail "build-metadata version wrongly failed"; fi rm -rf "$repo" -# PROSE / CODE-EXAMPLE MENTION: the bumped version's `## [1.1.0]` appears only -# inside prose (an indented example line), never as a line-start heading -> -# fails as UNDOCUMENTED. Proves the heading match is anchored, so a mention in -# body text or a fenced example neither satisfies nor falsely pre-exists the -# release entry. +# PROSE MENTION: the bumped version's `## [1.1.0]` appears only inline and on an +# indented line, never as a column-zero heading -> fails as UNDOCUMENTED. Proves +# the heading match is anchored, so an inline or indented mention neither +# satisfies nor falsely pre-exists the release entry. (The fenced-block case, +# where `## [1.1.0]` sits at column zero inside ```, is covered separately.) repo="$(mk_repo)" git_init "$repo" mk_plugin "$repo" alpha 1.0.0 yes @@ -172,6 +172,24 @@ rc=$? if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "version string in prose/indented example does not satisfy the anchored heading match"; else fail "unanchored-mention not caught: rc=$rc out='$out'"; fi rm -rf "$repo" +# FENCED-BLOCK EXAMPLE: the bumped version's `## [1.1.0]` sits at column zero but +# INSIDE a ``` fenced block, never as a real heading -> fails as UNDOCUMENTED. +# Proves the heading match tracks fence state (column-zero alone is not enough). +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +# shellcheck disable=SC2016 # single quotes are deliberate: the backtick fence and \n are literal changelog bytes, not shell expansions +printf '# Changelog\n\nExample of a heading:\n\n```\n## [1.1.0]\n```\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "fenced-code example heading (column zero inside a fence) does not satisfy --check-bump"; else fail "fenced-block heading wrongly satisfied gate: rc=$rc out='$out'"; fi +rm -rf "$repo" + # SYNTHETIC UNDOCUMENTED BUMP: version changed, changelog edited but WITHOUT an # entry for the new version (unrelated edit) -> fails. Proves the gate checks # the version's own entry, not merely that the file was touched. diff --git a/scripts/check-orphaned-fixtures.sh b/scripts/check-orphaned-fixtures.sh index 55965ce5d..ac10e7613 100755 --- a/scripts/check-orphaned-fixtures.sh +++ b/scripts/check-orphaned-fixtures.sh @@ -106,7 +106,12 @@ consumed() { base_re="(^|[^A-Za-z0-9._-])${esc_base}([^A-Za-z0-9._-]|$)" if [[ -f "$evals_json" ]]; then - if grep -qF "$rel" "$evals_json"; then + # Exact path match: compare $rel against each JSON string value, not a + # substring of the whole file — an eval referencing evals/fixtures/valid.json.bak + # must NOT consume a new evals/fixtures/valid.json sibling (shorter $rel is a + # substring of the longer value). jq failure (missing/invalid json) falls + # through to the bounded-basename check below. + if jq -e --arg r "$rel" 'any(.. | strings; . == $r)' "$evals_json" >/dev/null 2>&1; then return 0 fi if grep -qE "$base_re" "$evals_json"; then diff --git a/scripts/check-orphaned-fixtures.test.sh b/scripts/check-orphaned-fixtures.test.sh index 1a31c6486..21ed3546e 100755 --- a/scripts/check-orphaned-fixtures.test.sh +++ b/scripts/check-orphaned-fixtures.test.sh @@ -72,6 +72,19 @@ rc=$? if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"valid.json.bak"* && "$out" != *"ORPHANED FIXTURE"*"fixtures/valid.json is"* ]]; then ok "suffix sibling of a referenced fixture red-lines (bounded basename match)"; else fail "reference-name suffix sibling not caught: rc=$rc out='$out'"; fi rm -rf "$repo" +# --- files[] PATH match is EXACT, not a whole-file substring: an eval that +# references the longer evals/fixtures/valid.json.bak must NOT consume a new +# evals/fixtures/valid.json sibling (shorter $rel is a substring of the longer +# files[] value) -> valid.json red-lines, valid.json.bak stays consumed ------- +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '"evals/fixtures/valid.json.bak"' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json.bak" +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"fixtures/valid.json is"* && "$out" != *"valid.json.bak is"* ]]; then ok "files[] path match is exact (substring of a longer referenced path does not consume)"; else fail "files[] substring path match not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + # --- SYNTHETIC ORPHAN: consumed by nothing -> --check fails ---------------- repo="$(mk_repo)" seed_skill "$repo" "plugins/p/skills/s" '' From 0c1bcc309816c89d4539d106962833819aa5dd2b Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:45:24 -0400 Subject: [PATCH 10/12] =?UTF-8?q?fix(ci):=20close=20two=20review=20gaps=20?= =?UTF-8?q?=E2=80=94=20files[]-only=20consumption=20+=20CommonMark=20fence?= =?UTF-8?q?=20closing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check-orphaned-fixtures.sh: evals.json consumption now reads only files[] values (any nesting) — a fixture named in a prompt or metadata string no longer counts as consumed; exact-value and bounded-basename matches run against those values only. check-changelog-parity.sh: fence tracking records the opening delimiter char + length and closes only on a matching delimiter per CommonMark, so a ~~~ content line inside a backtick fence no longer re-opens heading matching. One regression test each. Co-Authored-By: Claude Fable 5 --- scripts/check-changelog-parity.sh | 24 +++++++++++++++++++----- scripts/check-changelog-parity.test.sh | 19 +++++++++++++++++++ scripts/check-orphaned-fixtures.sh | 19 +++++++++++-------- scripts/check-orphaned-fixtures.test.sh | 13 +++++++++++++ 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/scripts/check-changelog-parity.sh b/scripts/check-changelog-parity.sh index 2832829b3..bd7d857b8 100755 --- a/scripts/check-changelog-parity.sh +++ b/scripts/check-changelog-parity.sh @@ -143,14 +143,28 @@ for manifest in "${manifests[@]}"; do # changed: an unrelated edit (whitespace, title, an old release) must not # satisfy the gate. The match is a FIXED-STRING heading anchored to line # start (awk index()==1) and OUTSIDE fenced code, so the version string - # appearing in prose, an indented line, or a ``` / ~~~ example never - # satisfies — or falsely pre-exists — the release entry, and SemVer - # metacharacters (1.0.1+build.1) never leak into a regex. + # appearing in prose, an indented line, or a fenced example never satisfies — + # or falsely pre-exists — the release entry, and SemVer metacharacters + # (1.0.1+build.1) never leak into a regex. Fence tracking records the opening + # delimiter char and length and closes only on a matching delimiter (per + # CommonMark), so a mismatched inner marker — a ~~~ line inside a ``` block, + # or a shorter run of the same char — does not prematurely re-open the heading. heading="## [${head_version}]" has_heading() { awk -v h="$heading" ' - /^[[:space:]]*(```|~~~)/ { infence = !infence; next } - !infence && index($0, h) == 1 { found = 1; exit } + { + line = $0 + sub(/^[[:space:]]*/, "", line) + mchar = ""; mlen = 0 + if (match(line, /^`+/) && RLENGTH >= 3) { mchar = "`"; mlen = RLENGTH } + else if (match(line, /^~+/) && RLENGTH >= 3) { mchar = "~"; mlen = RLENGTH } + if (mchar != "") { + if (!infence) { infence = 1; fchar = mchar; flen = mlen } + else if (mchar == fchar && mlen >= flen) { infence = 0 } + next + } + if (!infence && index($0, h) == 1) { found = 1; exit } + } END { exit !found } ' } diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh index 84421cc40..10eb8c216 100755 --- a/scripts/check-changelog-parity.test.sh +++ b/scripts/check-changelog-parity.test.sh @@ -190,6 +190,25 @@ rc=$? if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "fenced-code example heading (column zero inside a fence) does not satisfy --check-bump"; else fail "fenced-block heading wrongly satisfied gate: rc=$rc out='$out'"; fi rm -rf "$repo" +# MISMATCHED INNER FENCE DELIMITER: a backtick fence contains a ~~~ content line +# (a different delimiter, so NOT a close per CommonMark); a `## [1.1.0]` after it +# is still inside the backtick fence -> fails as UNDOCUMENTED. Proves fence +# tracking closes only on the opening delimiter, not any fence-looking line. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +# shellcheck disable=SC2016 # single quotes are deliberate: the backtick/tilde fences and \n are literal changelog bytes +printf '# Changelog\n\n```\n~~~\n## [1.1.0]\n```\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "mismatched inner fence delimiter (tilde line inside a backtick fence) does not prematurely close"; else fail "fence delimiter mismatch mis-toggled: rc=$rc out='$out'"; fi +rm -rf "$repo" + # SYNTHETIC UNDOCUMENTED BUMP: version changed, changelog edited but WITHOUT an # entry for the new version (unrelated edit) -> fails. Proves the gate checks # the version's own entry, not merely that the file was touched. diff --git a/scripts/check-orphaned-fixtures.sh b/scripts/check-orphaned-fixtures.sh index ac10e7613..2fb7df3e1 100755 --- a/scripts/check-orphaned-fixtures.sh +++ b/scripts/check-orphaned-fixtures.sh @@ -83,7 +83,7 @@ plugin_root_of() { # consumed -> 0 if some grader consumes it consumed() { local fixture="$1" - local fixtures_dir evals_dir skill_dir evals_json rel base plugin + local fixtures_dir evals_dir skill_dir evals_json rel base plugin files_values fixtures_dir="${fixture%/*}" # walk up to the nearest 'fixtures' segment (fixtures may nest a subdir) @@ -106,15 +106,18 @@ consumed() { base_re="(^|[^A-Za-z0-9._-])${esc_base}([^A-Za-z0-9._-]|$)" if [[ -f "$evals_json" ]]; then - # Exact path match: compare $rel against each JSON string value, not a - # substring of the whole file — an eval referencing evals/fixtures/valid.json.bak - # must NOT consume a new evals/fixtures/valid.json sibling (shorter $rel is a - # substring of the longer value). jq failure (missing/invalid json) falls - # through to the bounded-basename check below. - if jq -e --arg r "$rel" 'any(.. | strings; . == $r)' "$evals_json" >/dev/null 2>&1; then + # Consumption via evals.json is limited to files[] VALUES: a fixture named + # only in a prompt or unrelated metadata string is NOT consumed. Extract every + # files[] string (at any nesting), then match $rel by whole-value equality, or + # the basename bounded within a value (a files[] form that spells the path + # differently still names the file). A shorter $rel is not a substring of a + # longer value under -x. jq failure (missing/invalid json) -> empty -> falls + # through to the test-file check below. + files_values="$(jq -r '.. | objects | .files? // empty | .[]? | select(type == "string")' "$evals_json" 2>/dev/null)" + if printf '%s\n' "$files_values" | grep -qxF -- "$rel"; then return 0 fi - if grep -qE "$base_re" "$evals_json"; then + if printf '%s\n' "$files_values" | grep -qE -- "$base_re"; then return 0 fi fi diff --git a/scripts/check-orphaned-fixtures.test.sh b/scripts/check-orphaned-fixtures.test.sh index 21ed3546e..4b21a8dd5 100755 --- a/scripts/check-orphaned-fixtures.test.sh +++ b/scripts/check-orphaned-fixtures.test.sh @@ -85,6 +85,19 @@ rc=$? if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"fixtures/valid.json is"* && "$out" != *"valid.json.bak is"* ]]; then ok "files[] path match is exact (substring of a longer referenced path does not consume)"; else fail "files[] substring path match not caught: rc=$rc out='$out'"; fi rm -rf "$repo" +# --- evals.json consumption is limited to files[] VALUES: a fixture named only +# in a prompt/metadata string, with an empty files[], is NOT consumed -> orphan - +repo="$(mk_repo)" +mkdir -p "$repo/plugins/p/skills/s/evals/fixtures" +cat >"$repo/plugins/p/skills/s/evals/evals.json" <<'JSON' +{ "evals": [ { "id": 1, "prompt": "open evals/fixtures/prose-only.md and grade the summary", "files": [] } ] } +JSON +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/prose-only.md" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"prose-only.md"* ]]; then ok "fixture named only in an eval prompt (empty files[]) is an orphan (consumption scoped to files[])"; else fail "prose-mention wrongly consumed: rc=$rc out='$out'"; fi +rm -rf "$repo" + # --- SYNTHETIC ORPHAN: consumed by nothing -> --check fails ---------------- repo="$(mk_repo)" seed_skill "$repo" "plugins/p/skills/s" '' From 600f1d281e3e2114b80c3e72835154a98ec08fee Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 11:53:08 -0400 Subject: [PATCH 11/12] fix(ci): validate closing-fence suffix + indentation per CommonMark; appease typos gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A close now requires the same delimiter char, a run at least as long as the opener, a whitespace-only suffix, and at-most-three-space indentation — a ```not-a-close content line or a four-space-indented would-be closer no longer clears fence state, so a release heading inside that fence cannot satisfy the parity gate. Opening backtick fences reject an info string containing a backtick. Regression test added; test-message wording adjusted for the spell-check gate (hyphen-split false positive on a prefixed word). Co-Authored-By: Claude Fable 5 --- scripts/check-changelog-parity.sh | 34 ++++++++++++++++---------- scripts/check-changelog-parity.test.sh | 22 ++++++++++++++++- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/scripts/check-changelog-parity.sh b/scripts/check-changelog-parity.sh index bd7d857b8..706898fa2 100755 --- a/scripts/check-changelog-parity.sh +++ b/scripts/check-changelog-parity.sh @@ -145,23 +145,31 @@ for manifest in "${manifests[@]}"; do # start (awk index()==1) and OUTSIDE fenced code, so the version string # appearing in prose, an indented line, or a fenced example never satisfies — # or falsely pre-exists — the release entry, and SemVer metacharacters - # (1.0.1+build.1) never leak into a regex. Fence tracking records the opening - # delimiter char and length and closes only on a matching delimiter (per - # CommonMark), so a mismatched inner marker — a ~~~ line inside a ``` block, - # or a shorter run of the same char — does not prematurely re-open the heading. + # (1.0.1+build.1) never leak into a regex. Fence tracking follows CommonMark: + # a fence line is a run of >=3 backticks or tildes indented at most three + # SPACES (four-plus, or a tab, is indented code); an opening backtick fence + # rejects an info string containing a backtick; a CLOSE requires the same + # delimiter char, a run at least as long as the opener, and nothing but + # whitespace after it — so a ~~~ line, a ```not-a-close line, or an indented + # would-be closer inside a ``` block never prematurely re-opens the heading. heading="## [${head_version}]" has_heading() { awk -v h="$heading" ' { - line = $0 - sub(/^[[:space:]]*/, "", line) - mchar = ""; mlen = 0 - if (match(line, /^`+/) && RLENGTH >= 3) { mchar = "`"; mlen = RLENGTH } - else if (match(line, /^~+/) && RLENGTH >= 3) { mchar = "~"; mlen = RLENGTH } - if (mchar != "") { - if (!infence) { infence = 1; fchar = mchar; flen = mlen } - else if (mchar == fchar && mlen >= flen) { infence = 0 } - next + if (match($0, /^ {0,3}`+/) || match($0, /^ {0,3}~+/)) { + seg = substr($0, RSTART, RLENGTH) + sub(/^ +/, "", seg) + mchar = substr(seg, 1, 1) + mlen = length(seg) + rest = substr($0, RSTART + RLENGTH) + if (mlen >= 3) { + if (!infence) { + # opening fence; a backtick info string must not contain a backtick + if (!(mchar == "`" && rest ~ /`/)) { infence = 1; fchar = mchar; flen = mlen; next } + } else if (mchar == fchar && mlen >= flen && rest ~ /^[ \t]*$/) { + infence = 0; next + } + } } if (!infence && index($0, h) == 1) { found = 1; exit } } diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh index 10eb8c216..8ca977bb6 100755 --- a/scripts/check-changelog-parity.test.sh +++ b/scripts/check-changelog-parity.test.sh @@ -206,7 +206,27 @@ printf '# Changelog\n\n```\n~~~\n## [1.1.0]\n```\n\n## [1.0.0]\n' >"$repo/plugin git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" rc=$? -if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "mismatched inner fence delimiter (tilde line inside a backtick fence) does not prematurely close"; else fail "fence delimiter mismatch mis-toggled: rc=$rc out='$out'"; fi +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "mismatched inner fence delimiter (tilde line inside a backtick fence) does not prematurely close"; else fail "fence delimiter mismatch wrongly toggled: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# NON-CLOSING FENCE LINES: a backtick fence contains a ```not-a-close line (text +# after the run) and a four-space-indented ``` line (indented code column, not a +# fence per CommonMark); neither closes, so a `## [1.1.0]` after them is still +# fenced -> fails as UNDOCUMENTED. Proves a close requires a whitespace-only +# suffix and at-most-three-space indentation. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +# shellcheck disable=SC2016 # single quotes are deliberate: fence lines are literal changelog bytes +printf '# Changelog\n\n```\n```not-a-close\n ```\n## [1.1.0]\n```\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "non-closing fence lines (text suffix, four-space indent) do not close a fence"; else fail "non-closing fence line wrongly closed the fence: rc=$rc out='$out'"; fi rm -rf "$repo" # SYNTHETIC UNDOCUMENTED BUMP: version changed, changelog edited but WITHOUT an From efb9717392da158fcf5a30570b9ee4bad43558f8 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Mon, 20 Jul 2026 12:02:59 -0400 Subject: [PATCH 12/12] fix(ci): scope test-basename consumption to the owning skill; ignore HTML-comment headings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit check-orphaned-fixtures.sh: a bounded-basename match in a test file counts only inside the fixture's own skill; a test elsewhere in the plugin must name the fixture by its plugin-relative path — same-named fixtures in sibling skills are no longer conflated. check-changelog-parity.sh: has_heading tracks multi-line HTML-comment state alongside fences — a release heading inside is not rendered Markdown and neither satisfies nor pre-exists the entry; comment markers inside fences stay content, fence markers inside comments stay suppressed. Regression tests for all three shapes. Co-Authored-By: Claude Fable 5 --- scripts/check-changelog-parity.sh | 27 +++++++++++++++++++++++++ scripts/check-changelog-parity.test.sh | 16 +++++++++++++++ scripts/check-orphaned-fixtures.sh | 16 +++++++++++++-- scripts/check-orphaned-fixtures.test.sh | 25 +++++++++++++++++++++++ 4 files changed, 82 insertions(+), 2 deletions(-) diff --git a/scripts/check-changelog-parity.sh b/scripts/check-changelog-parity.sh index 706898fa2..4a34ccbaf 100755 --- a/scripts/check-changelog-parity.sh +++ b/scripts/check-changelog-parity.sh @@ -152,10 +152,28 @@ for manifest in "${manifests[@]}"; do # delimiter char, a run at least as long as the opener, and nothing but # whitespace after it — so a ~~~ line, a ```not-a-close line, or an indented # would-be closer inside a ``` block never prematurely re-opens the heading. + # HTML raw blocks are tracked alongside fences: a heading inside a multi-line + # comment is not rendered Markdown, so it neither satisfies nor + # pre-exists the release entry. A same-line "" can never match + # anyway (the heading is not at column one). Comment markers inside fenced + # code are content; fence markers inside a comment are suppressed. heading="## [${head_version}]" has_heading() { awk -v h="$heading" ' { + if (!infence && inhtml) { + p = index($0, "-->") + if (p == 0) next + inhtml = 0 + rem = substr($0, p + 3) + while ((q = index(rem, "") + if (r == 0) { inhtml = 1; break } + rem = substr(rem, r + 3) + } + next + } if (match($0, /^ {0,3}`+/) || match($0, /^ {0,3}~+/)) { seg = substr($0, RSTART, RLENGTH) sub(/^ +/, "", seg) @@ -172,6 +190,15 @@ for manifest in "${manifests[@]}"; do } } if (!infence && index($0, h) == 1) { found = 1; exit } + if (!infence) { + rem = $0 + while ((q = index(rem, "") + if (r == 0) { inhtml = 1; break } + rem = substr(rem, r + 3) + } + } } END { exit !found } ' diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh index 8ca977bb6..12b245ade 100755 --- a/scripts/check-changelog-parity.test.sh +++ b/scripts/check-changelog-parity.test.sh @@ -229,6 +229,22 @@ rc=$? if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "non-closing fence lines (text suffix, four-space indent) do not close a fence"; else fail "non-closing fence line wrongly closed the fence: rc=$rc out='$out'"; fi rm -rf "$repo" +# HTML-COMMENT HEADING: the bumped version's heading appears only inside a +# multi-line comment -> not rendered Markdown -> fails as UNDOCUMENTED. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "heading inside an HTML comment does not satisfy --check-bump"; else fail "HTML-comment heading wrongly satisfied gate: rc=$rc out='$out'"; fi +rm -rf "$repo" + # SYNTHETIC UNDOCUMENTED BUMP: version changed, changelog edited but WITHOUT an # entry for the new version (unrelated edit) -> fails. Proves the gate checks # the version's own entry, not merely that the file was touched. diff --git a/scripts/check-orphaned-fixtures.sh b/scripts/check-orphaned-fixtures.sh index 2fb7df3e1..b65b397d4 100755 --- a/scripts/check-orphaned-fixtures.sh +++ b/scripts/check-orphaned-fixtures.sh @@ -83,7 +83,7 @@ plugin_root_of() { # consumed -> 0 if some grader consumes it consumed() { local fixture="$1" - local fixtures_dir evals_dir skill_dir evals_json rel base plugin files_values + local fixtures_dir evals_dir skill_dir evals_json rel base plugin plugin_rel files_values fixtures_dir="${fixture%/*}" # walk up to the nearest 'fixtures' segment (fixtures may nest a subdir) @@ -122,11 +122,23 @@ consumed() { fi fi - plugin="$(plugin_root_of "$fixture")" + # Test-file consumption is scoped: a bounded-basename match counts only inside + # the OWNING skill, so same-named fixtures in sibling skills are never + # conflated. A test elsewhere in the plugin must name the fixture by its + # plugin-relative path (fixed string), which is unambiguous across skills. while IFS= read -r -d '' test_file; do if grep -qE "$base_re" "$test_file"; then return 0 fi + done < <(find "$skill_dir" -type f -name '*.test.*' -print0 2>/dev/null) + + plugin="$(plugin_root_of "$fixture")" + plugin_rel="${fixture#"$plugin"/}" + while IFS= read -r -d '' test_file; do + case "$test_file" in "$skill_dir"/*) continue ;; *) ;; esac + if grep -qF -- "$plugin_rel" "$test_file"; then + return 0 + fi done < <(find "$plugin" -type f -name '*.test.*' -print0 2>/dev/null) return 1 diff --git a/scripts/check-orphaned-fixtures.test.sh b/scripts/check-orphaned-fixtures.test.sh index 4b21a8dd5..a8952534b 100755 --- a/scripts/check-orphaned-fixtures.test.sh +++ b/scripts/check-orphaned-fixtures.test.sh @@ -85,6 +85,31 @@ rc=$? if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"fixtures/valid.json is"* && "$out" != *"valid.json.bak is"* ]]; then ok "files[] path match is exact (substring of a longer referenced path does not consume)"; else fail "files[] substring path match not caught: rc=$rc out='$out'"; fi rm -rf "$repo" +# --- SIBLING-SKILL BASENAME CONFLATION: skill a's test naming shared.md must +# NOT consume skill b's same-named fixture (basename matches count only inside +# the owning skill; cross-skill consumption needs the plugin-relative path) ---- +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/a" '' +seed_skill "$repo" "plugins/p/skills/b" '' +printf 'x\n' >"$repo/plugins/p/skills/a/evals/fixtures/shared.md" +printf 'x\n' >"$repo/plugins/p/skills/b/evals/fixtures/shared.md" +mkdir -p "$repo/plugins/p/skills/a/scripts" +printf 'assert_on fixtures/shared.md\n' >"$repo/plugins/p/skills/a/scripts/thing.test.sh" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"skills/b/evals/fixtures/shared.md"* && "$out" != *"skills/a/evals/fixtures/shared.md is"* ]]; then ok "sibling skill's same-named fixture is not conflated (test basename scoped to owning skill)"; else fail "sibling-skill basename conflation not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# --- CROSS-SKILL CONSUMPTION BY PLUGIN-RELATIVE PATH: a plugin-root test naming +# skills/b/evals/fixtures/by-plugin-test.md consumes it ------------------------ +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/b" '' +printf 'x\n' >"$repo/plugins/p/skills/b/evals/fixtures/by-plugin-test.md" +mkdir -p "$repo/plugins/p/tools" +printf 'assert_on skills/b/evals/fixtures/by-plugin-test.md\n' >"$repo/plugins/p/tools/suite.test.sh" +if run_check "$repo" >/dev/null; then ok "plugin-root test consuming by plugin-relative path passes --check"; else fail "plugin-relative path consumption wrongly flagged"; fi +rm -rf "$repo" + # --- evals.json consumption is limited to files[] VALUES: a fixture named only # in a prompt/metadata string, with an empty files[], is NOT consumed -> orphan - repo="$(mk_repo)"