diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index daeb040fe..dcd906cdb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -295,6 +295,53 @@ jobs: - name: Run silent-skip-gate tests run: bash scripts/check-silent-skips.test.sh + # Every file under a skill's evals/fixtures/ must be consumed by a grader — an + # eval files[] entry or a test assertion — so an ungraded suite cannot sit in + # the tree reading as tested. The check is repo-wide and static; existing debt + # is grandfathered in scripts/orphaned-fixtures-baseline.txt with a stale + # guard, so the gate holds the line without red-lining fixtures a member issue + # already owns. The self-test runs unconditionally so a broken detector cannot + # mask a regression behind a green gate. + orphaned-fixture-gate: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Check out + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Test the orphaned-fixture detector + run: bash scripts/check-orphaned-fixtures.test.sh + - name: Check for orphaned eval fixtures + run: scripts/check-orphaned-fixtures.sh --check + + # CHANGELOG parity: a versioned plugin must keep a CHANGELOG.md (static + # --check), and a PR that changes a plugin's manifest version must update that + # plugin's CHANGELOG.md in the same diff (--check-bump, PR-only). Existing + # "versioned but changelog-less" debt is grandfathered by name in + # scripts/changelog-parity-baseline.txt with a stale guard; the bump gate is + # never relaxed by the baseline. The self-test runs unconditionally so a broken + # gate cannot mask a regression, and the PR-diff step is event-gated. + changelog-parity-gate: + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Check out + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + # Full history so the PR base ref is resolvable for the bump gate. + fetch-depth: 0 + - name: Test the changelog-parity gate + run: bash scripts/check-changelog-parity.test.sh + - name: Verify every versioned plugin keeps a CHANGELOG.md + run: scripts/check-changelog-parity.sh --check + - name: Verify a version bump updates the plugin's CHANGELOG.md + if: github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.base_ref }} + run: scripts/check-changelog-parity.sh --check-bump "origin/$BASE_REF" + # Deep plugin-contract lane and the heaviest suite (Node + Python installs, # every plugins/**/*.test.sh, manifest + catalog validation). A PR whose diff # is confined to the docs-only allowlist (scripts/docs-only-paths.txt) cannot @@ -531,6 +578,8 @@ jobs: - standards-contract-sync - cross-plugin-source-drift - silent-skip-gate + - orphaned-fixture-gate + - changelog-parity-gate - plugin-gate - miro-plugin - runner-policy diff --git a/plugins/discovery/CHANGELOG.md b/plugins/discovery/CHANGELOG.md index 72ddaf591..3d304e738 100644 --- a/plugins/discovery/CHANGELOG.md +++ b/plugins/discovery/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog — discovery plugin -## 0.7.3 — 2026-07-19 +## [0.7.3] — 2026-07-19 ### Fixed @@ -11,7 +11,7 @@ current working directory when no repo root exists. The live root stays available for resolving files while working; it is never echoed into the handoff, so `EXPLORE.md` stays machine-agnostic. -## 0.7.2 — 2026-07-19 +## [0.7.2] — 2026-07-19 ### Changed @@ -21,7 +21,7 @@ write path — replacing the dead `docs/adr/…` reference that resolves to nothing in the cache-isolated installed plugin. Behavior is unchanged; gitbook stays deferred and non-writable. -## 0.7.0 — 2026-07-18 +## [0.7.0] — 2026-07-18 ### Changed @@ -34,7 +34,7 @@ interview, so headless and CI use are possible. The ignore guard, the gitbook-deferred handling, and the never-edit-root-`.gitignore` rule are unchanged. -## 0.6.0 — 2026-07-17 +## [0.6.0] — 2026-07-17 ### Changed @@ -45,7 +45,7 @@ artifacts there directly; only workers dispatched into their own checkout return findings by value for the parent to write. -## 0.5.1 — 2026-07-15 +## [0.5.1] — 2026-07-15 ### Fixed @@ -54,7 +54,7 @@ and states that durable writes still target `docs`; the skill never configures or tests a GitBook API, MCP, or Git Sync writer. -## 0.5.0 — 2026-07-15 +## [0.5.0] — 2026-07-15 ### Added @@ -65,7 +65,7 @@ doubled phase/query/source minimums (SKILL.md item 8, discipline.md's "Broad-topic auto-detect"). -## 0.4.0 — 2026-07-14 +## [0.4.0] — 2026-07-14 Adopt the marketplace topic-docs convention, contract v1.0.0 (): diff --git a/plugins/docs-hygiene/CHANGELOG.md b/plugins/docs-hygiene/CHANGELOG.md index dbf68503d..29b1894ea 100644 --- a/plugins/docs-hygiene/CHANGELOG.md +++ b/plugins/docs-hygiene/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog — docs-hygiene plugin -## 0.7.0 — 2026-07-18 +## [0.7.0] — 2026-07-18 Changed: @@ -12,7 +12,7 @@ Changed: ambient, Git Bash on native Windows), the compress-only `markdownlint-cli2` requirement with its absence behavior, and the optional `caveman` backend. -## 0.6.0 — 2026-07-17 +## [0.6.0] — 2026-07-17 Changed: @@ -24,7 +24,7 @@ Changed: word. The detect-script env vars moved with it: `DECLUTTER_REPO_ROOT` → `AUDIT_NOISE_REPO_ROOT`. -## 0.5.0 — 2026-07-15 +## [0.5.0] — 2026-07-15 Changed: @@ -32,7 +32,7 @@ Changed: `/docs-hygiene:encapsulation-audit` invocations to `/docs-hygiene:audit-encapsulation`; the plugin ID (`docs-hygiene`) is unchanged, only the skill's leaf name moved. -## 0.4.0 — 2026-07-15 +## [0.4.0] — 2026-07-15 Added: @@ -44,7 +44,7 @@ Added: - The "add an eval case" clause, re-added to the two Gotchas/Recheck-trigger bullets in rename-references/SKILL.md. -## 0.3.0 — 2026-07-14 +## [0.3.0] — 2026-07-14 Adopt the marketplace topic-docs convention (`docs/conventions/topic-docs/`, contract v1.0.0) in the declutter diff --git a/plugins/knowledge/CHANGELOG.md b/plugins/knowledge/CHANGELOG.md index 10f35f91c..5ff9f1458 100644 --- a/plugins/knowledge/CHANGELOG.md +++ b/plugins/knowledge/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to the `knowledge` plugin are recorded here. The `version` i `.claude-plugin/plugin.json` is the delivery vehicle — a consumer receives a change only after that version increases. -## 0.8.0 +## [0.8.0] ### Changed @@ -18,7 +18,7 @@ only after that version increases. scripts the ingest skills already run — pulling the prerequisite/provisioning surface onto the setup contract. The personal env-channel scalars are unchanged. -## 0.7.1 +## [0.7.1] ### Changed @@ -27,7 +27,7 @@ only after that version increases. the EPUB branch's `unzip` requirement (not bundled with Git Bash) — cross-platform declaration wave. PDF-only use needs neither extra install. -## 0.7.0 +## [0.7.0] ### Changed @@ -54,13 +54,13 @@ only after that version increases. plaintext in `~/.claude/.credentials.json` on Windows, so those secrets stay in shell env until keychain-backed sensitive storage lands there (documented at the course-digest auth-env site). -## 0.6.1 +## [0.6.1] ### Changed - References to the renamed `/planning:plan` skill (was `/planning:architect`, planning 0.13.0 breaking rename) retargeted. Version bumped so existing installs receive the rewritten prompts. -## 0.6.0 +## [0.6.0] ### Changed @@ -72,7 +72,7 @@ only after that version increases. unaffected (`.work//...` layout is unchanged); resume with `/knowledge:youtube-digest resume `. -## 0.5.6 +## [0.5.6] ### Added @@ -80,7 +80,7 @@ only after that version increases. graduate into a standalone plugin (once its vendored `video-digestion` package is independently distributable). -## 0.5.5 +## [0.5.5] ### Fixed @@ -88,7 +88,7 @@ only after that version increases. linear pass, but now preserves an unmatched literal less-than tail instead of truncating the rest of programming, mathematics, or other tolerant transcript text. -## 0.5.4 +## [0.5.4] ### Fixed @@ -101,7 +101,7 @@ only after that version increases. clone option parsing is terminated explicitly; and WebVTT/entity cleanup uses bounded, single-pass transformations that cannot turn nested malformed input into active markup. -## 0.5.3 +## [0.5.3] ### Changed @@ -115,7 +115,7 @@ only after that version increases. now records that the `verification/` lane name matches the convention's canon. Docs-only — no paths or behavior change; the `library_dir` seam is untouched. -## 0.5.2 +## [0.5.2] ### Fixed @@ -141,7 +141,7 @@ only after that version increases. forces `--no-playlist`; and a passing key-frame quality-audit row now requires a substantive evidence note (an omitted note no longer bypasses the gate). -## 0.5.1 +## [0.5.1] ### Changed @@ -153,7 +153,7 @@ only after that version increases. into `${CLAUDE_PLUGIN_DATA}` is unchanged. Internal restructure — no consumer-facing behavior change; the version bump delivers the moved source (and the new install fingerprint) to consumers. -## 0.5.0 +## [0.5.0] ### Added @@ -189,7 +189,7 @@ only after that version increases. - `repo-analysis` and `video-digestion` are now vendored by both the youtube and course-digest skills. Deduplication of the two copies is tracked separately. -## 0.4.0 +## [0.4.0] ### Changed @@ -208,7 +208,7 @@ only after that version increases. land, restoring the stronger wording softened while the seam was unwired (`book-distill` remains the documented exception). -## 0.3.0 +## [0.3.0] ### Changed @@ -216,7 +216,7 @@ only after that version increases. hardening so synthesized artifacts land at the configured library directory in the consuming repo. -## 0.2.0 +## [0.2.0] ### Added @@ -237,6 +237,6 @@ only after that version increases. dependencies resolve from `${CLAUDE_PLUGIN_DATA}` (NODE_PATH is CommonJS-only and does not apply). -## 0.1.1 +## [0.1.1] - Prior baseline: `book-distill` and `setup` skills. diff --git a/plugins/playbooks/CHANGELOG.md b/plugins/playbooks/CHANGELOG.md index 9ea486e09..f2f3623d4 100644 --- a/plugins/playbooks/CHANGELOG.md +++ b/plugins/playbooks/CHANGELOG.md @@ -4,7 +4,7 @@ All notable changes to the `playbooks` plugin are recorded here. The `version` i `.claude-plugin/plugin.json` is the delivery vehicle — a consumer receives a change only after that version increases. -## 0.2.0 +## [0.2.0] ### Changed @@ -17,7 +17,7 @@ only after that version increases. drift-check mechanics now point at the renamed pack path. Only the wrapper skill name (directory, frontmatter `name`, and references) changed. -## 0.1.0 +## [0.1.0] ### Added diff --git a/plugins/session-flow/CHANGELOG.md b/plugins/session-flow/CHANGELOG.md index e6c4bdab8..1e02c5a1a 100644 --- a/plugins/session-flow/CHANGELOG.md +++ b/plugins/session-flow/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog — session-flow plugin -## 0.10.2 — 2026-07-19 +## [0.10.2] — 2026-07-19 Fixed: @@ -10,7 +10,7 @@ Fixed: copy-as-is run of the snippet previously bypassed the memory_dir seam, missing the handoff chain in any repo that relocates its memory tier. -## 0.10.1 — 2026-07-19 +## [0.10.1] — 2026-07-19 Changed: @@ -20,7 +20,7 @@ Changed: own sections and keeps only the plugin-specific no-project-root fallback detail. -## 0.10.0 — 2026-07-18 +## [0.10.0] — 2026-07-18 Added: @@ -38,7 +38,7 @@ Added: worktree mechanics route to whatever capabilities are installed, falling back to direct git / gh. The plugin now bundles seven skills. -## 0.9.1 — 2026-07-18 +## [0.9.1] — 2026-07-18 Fixed: @@ -50,7 +50,7 @@ Fixed: model defaults to `inherit` per the subagents doc (resolution order and cost-control quote now cited in `context/sources.md`). -## 0.9.0 — 2026-07-18 +## [0.9.0] — 2026-07-18 Added: @@ -63,7 +63,7 @@ Added: re-anchors; it does not resume the work (the keep-going sibling), enumerate worktrees, or triage PR feedback. The plugin now bundles six skills. -## 0.8.0 — 2026-07-17 +## [0.8.0] — 2026-07-17 Changed: @@ -72,13 +72,13 @@ Changed: the contract's `.worktreeinclude` template carries into new worktrees, while handoffs are session-scoped and deliberately not carried. -## 0.7.1 +## [0.7.1] ### Changed - References to the renamed `/planning:plan` skill (was `/planning:architect`, planning 0.13.0 breaking rename) retargeted. Version bumped so existing installs receive the rewritten prompts. -## 0.7.0 — 2026-07-17 +## [0.7.0] — 2026-07-17 Added: @@ -93,7 +93,7 @@ Added: the interruption cause is deliberately not diagnosed (recovery is identical regardless). The plugin now bundles five skills. -## 0.6.0 — 2026-07-16 +## [0.6.0] — 2026-07-16 Changed: @@ -113,7 +113,7 @@ Added: pressure, and concurrent-session / rate-limit headroom, with small/medium/large fan-out sizing and single-agent as the floor. -## 0.5.0 — 2026-07-15 +## [0.5.0] — 2026-07-15 Added: @@ -145,7 +145,7 @@ Added: actual capability inventory whenever capabilities are added, renamed, or retired. -## 0.4.0 — 2026-07-15 +## [0.4.0] — 2026-07-15 Added: @@ -159,7 +159,7 @@ Added: handoff without verifiable sanity-check evidence, continuing past an explicit stop). Loaded on demand from the SKILL.md checklist. -## 0.3.0 — 2026-07-14 +## [0.3.0] — 2026-07-14 Adopt the marketplace topic-docs convention (`docs/conventions/topic-docs/`, contract v1.0.0): diff --git a/scripts/changelog-parity-baseline.txt b/scripts/changelog-parity-baseline.txt new file mode 100644 index 000000000..3898ba24c --- /dev/null +++ b/scripts/changelog-parity-baseline.txt @@ -0,0 +1,12 @@ +# Grandfathered "versioned plugin with no CHANGELOG.md" debt — existing gaps +# check-changelog-parity.sh --check must not red-line. One plugin NAME per line +# (comments and blank lines ignored). Stale-guarded: --check fails if a listed +# plugin gains a CHANGELOG.md or drops its version, so an entry cannot outlive +# its debt. This never relaxes --check-bump — a listed plugin that bumps its +# version still must start its changelog in that same PR. +# +# autonomy: shipped 5 minor version bumps with no CHANGELOG.md while every other +# plugin maintains one — the parity gap this gate exists to flag. Reconstructing +# an accurate history is autonomy-plugin (WP-lane) work, not this repo-level CI +# change; grandfathered pending a dedicated autonomy CHANGELOG issue. +autonomy diff --git a/scripts/check-changelog-parity.sh b/scripts/check-changelog-parity.sh new file mode 100755 index 000000000..4a34ccbaf --- /dev/null +++ b/scripts/check-changelog-parity.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +# Keep each plugin's CHANGELOG.md honest against its manifest version. +# +# scripts/check-changelog-parity.sh --check fail if a versioned +# plugin has no sibling +# CHANGELOG.md +# scripts/check-changelog-parity.sh --check-bump fail if a plugin's +# manifest version +# changed vs but +# its CHANGELOG.md does +# not ADD a `## []` +# entry for the new +# version (present at +# head, absent at ) +# +# Two complementary gaps the same audit surfaced: +# * --check is the static repo-wide invariant: a plugins//.claude-plugin/ +# plugin.json carrying a `version` must ship a plugins//CHANGELOG.md. +# It catches a plugin that has bumped versions but never kept a changelog at +# all (autonomy shipped 5 minor bumps with none). +# * --check-bump is the go-forward PR discipline: a version change must ADD a +# `## []` entry for the new version — present in the plugin's +# CHANGELOG.md at head AND absent from it at . Two failure classes the +# gate must not conflate: (a) checking for the version's own entry, not +# merely that the file was touched, so an unrelated edit — whitespace, the +# title, an old release — cannot satisfy the gate; (b) requiring the entry +# to be newly added, so reusing a heading that already existed at +# (a bump with no fresh release note) cannot satisfy it either. An entry +# documented as `## ` (no brackets) is a separate FORMAT failure, +# reported as such rather than as undocumented. It applies to EVERY plugin, +# grandfathered or not — the moment a debt-listed plugin bumps again it must +# start its changelog. +# +# Existing "versioned but changelog-less" debt is grandfathered by plugin NAME in +# scripts/changelog-parity-baseline.txt (same stale-guarded idiom as +# scripts/orphaned-fixtures-baseline.txt): --check exempts a listed plugin but +# fails on a STALE entry — one that now has a CHANGELOG.md (or no version) — so +# the exemption cannot outlive the debt. The baseline never relaxes --check-bump. +# +# Fail-closed: a versioned plugin with neither a CHANGELOG.md nor a baseline +# entry fails. CHANGELOG_PARITY_BASELINE overrides the baseline path (test +# injection). +set -uo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 2 + +BASELINE="${CHANGELOG_PARITY_BASELINE:-scripts/changelog-parity-baseline.txt}" + +mode="${1:-}" +case "$mode" in +--check | --check-bump) ;; +*) + echo "usage: $(basename "$0") [--check | --check-bump ]" >&2 + exit 2 + ;; +esac + +# Grandfathered plugin names (static-check exemptions). +declare -A grandfathered +if [[ -f "$BASELINE" ]]; then + while IFS= read -r line; do + line="${line%%#*}" + line="${line#"${line%%[![:space:]]*}"}" + line="${line%"${line##*[![:space:]]}"}" + [[ -z "$line" ]] && continue + grandfathered["$line"]=1 + done <"$BASELINE" +fi + +manifests=(plugins/*/.claude-plugin/plugin.json) +if [[ ! -e "${manifests[0]}" ]]; then + echo "error: no plugin manifests found under plugins/*/.claude-plugin/" >&2 + exit 2 +fi + +version_of() { jq -r '.version // empty' "$1" 2>/dev/null; } + +if [[ "$mode" == "--check" ]]; then + declare -A saw_debt + missing=0 + for manifest in "${manifests[@]}"; do + plugin_dir="${manifest%/.claude-plugin/plugin.json}" + name="${plugin_dir##*/}" + [[ -n "$(version_of "$manifest")" ]] || continue + if [[ -f "$plugin_dir/CHANGELOG.md" ]]; then + if [[ -n "${grandfathered[$name]:-}" ]]; then + echo "STALE BASELINE: '$name' in $BASELINE now has a CHANGELOG.md — remove it." >&2 + missing=$((missing + 1)) + # Mark handled so the second stale-scan loop does not re-report it. + saw_debt["$name"]=1 + fi + continue + fi + if [[ -n "${grandfathered[$name]:-}" ]]; then + saw_debt["$name"]=1 + continue + fi + echo "MISSING CHANGELOG: $plugin_dir carries a versioned $manifest but no $plugin_dir/CHANGELOG.md." >&2 + echo " Add $plugin_dir/CHANGELOG.md, or grandfather '$name' in $BASELINE with its owning issue." >&2 + missing=$((missing + 1)) + done + # A baseline name that matches no versioned-and-changelog-less plugin is stale. + for name in "${!grandfathered[@]}"; do + if [[ -z "${saw_debt[$name]:-}" ]]; then + echo "STALE BASELINE: '$name' in $BASELINE no longer names a versioned plugin missing a CHANGELOG.md — remove it." >&2 + missing=$((missing + 1)) + fi + done + if ((missing > 0)); then + exit 1 + fi + echo "Every versioned plugin has a CHANGELOG.md (or a stale-guarded baseline entry)." + exit 0 +fi + +# --check-bump mode +if [[ -z "${2:-}" ]]; then + echo "usage: $(basename "$0") --check-bump " >&2 + exit 2 +fi +base="$2" +if ! git rev-parse --verify --quiet "${base}^{commit}" >/dev/null; then + echo "check-changelog-parity: base ref '$base' is not a resolvable commit." >&2 + exit 2 +fi + +undocumented=0 +malformed=0 +preexisting=0 +for manifest in "${manifests[@]}"; do + plugin_dir="${manifest%/.claude-plugin/plugin.json}" + name="${plugin_dir##*/}" + changelog="$plugin_dir/CHANGELOG.md" + + base_version="$(git show "$base:$manifest" 2>/dev/null | jq -r '.version // empty' 2>/dev/null || true)" + # Absent at base => new plugin in this change set; the static --check owns + # whether its initial CHANGELOG.md exists, not the bump gate. + [[ -n "$base_version" ]] || continue + head_version="$(version_of "$manifest")" + [[ "$head_version" != "$base_version" ]] || continue + + # Require the bumped version's own entry at head, not merely that the file + # changed: an unrelated edit (whitespace, title, an old release) must not + # satisfy the gate. The match is a FIXED-STRING heading anchored to line + # start (awk index()==1) and OUTSIDE fenced code, so the version string + # appearing in prose, an indented line, or a fenced example never satisfies — + # or falsely pre-exists — the release entry, and SemVer metacharacters + # (1.0.1+build.1) never leak into a regex. Fence tracking follows CommonMark: + # a fence line is a run of >=3 backticks or tildes indented at most three + # SPACES (four-plus, or a tab, is indented code); an opening backtick fence + # rejects an info string containing a backtick; a CLOSE requires the same + # delimiter char, a run at least as long as the opener, and nothing but + # whitespace after it — so a ~~~ line, a ```not-a-close line, or an indented + # would-be closer inside a ``` block never prematurely re-opens the heading. + # HTML raw blocks are tracked alongside fences: a heading inside a multi-line + # comment is not rendered Markdown, so it neither satisfies nor + # pre-exists the release entry. A same-line "" can never match + # anyway (the heading is not at column one). Comment markers inside fenced + # code are content; fence markers inside a comment are suppressed. + heading="## [${head_version}]" + has_heading() { + awk -v h="$heading" ' + { + if (!infence && inhtml) { + p = index($0, "-->") + if (p == 0) next + inhtml = 0 + rem = substr($0, p + 3) + while ((q = index(rem, "") + if (r == 0) { inhtml = 1; break } + rem = substr(rem, r + 3) + } + next + } + if (match($0, /^ {0,3}`+/) || match($0, /^ {0,3}~+/)) { + seg = substr($0, RSTART, RLENGTH) + sub(/^ +/, "", seg) + mchar = substr(seg, 1, 1) + mlen = length(seg) + rest = substr($0, RSTART + RLENGTH) + if (mlen >= 3) { + if (!infence) { + # opening fence; a backtick info string must not contain a backtick + if (!(mchar == "`" && rest ~ /`/)) { infence = 1; fchar = mchar; flen = mlen; next } + } else if (mchar == fchar && mlen >= flen && rest ~ /^[ \t]*$/) { + infence = 0; next + } + } + } + if (!infence && index($0, h) == 1) { found = 1; exit } + if (!infence) { + rem = $0 + while ((q = index(rem, "") + if (r == 0) { inhtml = 1; break } + rem = substr(rem, r + 3) + } + } + } + END { exit !found } + ' + } + if [[ -f "$changelog" ]] && has_heading <"$changelog"; then + # The release entry must be ADDED by this change set, not merely present: + # a heading that already existed in the changelog at $base means the bump + # reused a pre-existing entry and shipped no new release note. Require it + # absent from the base changelog. (git show fails for a changelog that is + # new at $base -> empty -> counts as absent, which is correct: it was added + # here.) + if git show "$base:$changelog" 2>/dev/null | has_heading; then + echo "PRE-EXISTING CHANGELOG ENTRY: $name bumped $base_version -> $head_version but '## [$head_version]' already existed in $changelog at $base; add the release entry in this change set." >&2 + preexisting=$((preexisting + 1)) + fi + continue + fi + + # Split the failure: a heading that names the version but omits the brackets + # (## ) is a FORMAT error the author can fix in place, not a missing + # release. Escape ALL ERE metacharacters (SemVer allows + in build metadata), + # not just dots. + # shellcheck disable=SC2016 # single quotes are deliberate: $ is an ERE metachar being escaped, not a shell expansion + esc="$(printf '%s' "$head_version" | sed -E 's/[][\\.|$(){}?+*^]/\\&/g')" + if found="$([[ -f "$changelog" ]] && grep -m1 -E "^##[[:space:]]+${esc}([[:space:]]|\$)" "$changelog")"; then + echo "CHANGELOG FORMAT: $name $head_version is documented as '${found}' but must use the bracketed Keep-a-Changelog heading '## [$head_version]'." >&2 + malformed=$((malformed + 1)) + else + echo "UNDOCUMENTED BUMP: $name went $base_version -> $head_version but $changelog has no '## [$head_version]' entry at head." >&2 + undocumented=$((undocumented + 1)) + fi +done + +if ((undocumented > 0 || malformed > 0 || preexisting > 0)); then + ((undocumented > 0)) && echo "Add a '## []' entry for every plugin whose version changed." >&2 + ((malformed > 0)) && echo "Convert unbracketed changelog headings to the '## []' Keep-a-Changelog form." >&2 + ((preexisting > 0)) && echo "Add the bumped version's '## []' entry in this change set; it must be absent from the base changelog, not merely present at head." >&2 + exit 1 +fi +echo "Every plugin whose version changed vs $base has a '## []' CHANGELOG.md entry." diff --git a/scripts/check-changelog-parity.test.sh b/scripts/check-changelog-parity.test.sh new file mode 100755 index 000000000..12b245ade --- /dev/null +++ b/scripts/check-changelog-parity.test.sh @@ -0,0 +1,323 @@ +#!/usr/bin/env bash +# Unit tests for check-changelog-parity.sh. --check scenarios build a throwaway +# plugins/ tree; --check-bump scenarios build a real two-commit git history so +# the version-changed-but-changelog-untouched case is exercised end to end. The +# negative cases are the synthetic proof the gate catches an undocumented +# version bump and a versioned-but-changelog-less plugin. +set -uo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="$SELF_DIR/check-changelog-parity.sh" + +PASS=0 +FAIL=0 +fail() { + echo "FAIL: $*" >&2 + FAIL=$((FAIL + 1)) +} +ok() { + echo "ok: $*" + PASS=$((PASS + 1)) +} + +mk_repo() { + local dir + dir="$(mktemp -d)" + mkdir -p "$dir/scripts" + cp "$SCRIPT" "$dir/scripts/check-changelog-parity.sh" + printf '%s' "${1:-}" >"$dir/scripts/changelog-parity-baseline.txt" + printf '%s' "$dir" +} + +git_init() { + local dir="$1" + git -C "$dir" init -q + git -C "$dir" config user.email t@t.test + git -C "$dir" config user.name test + git -C "$dir" config commit.gpgsign false + git -C "$dir" config core.autocrlf false +} + +mk_plugin() { + local repo="$1" name="$2" version="$3" changelog="$4" + mkdir -p "$repo/plugins/$name/.claude-plugin" + printf '{ "name": "%s", "version": "%s" }\n' "$name" "$version" \ + >"$repo/plugins/$name/.claude-plugin/plugin.json" + [[ "$changelog" == "yes" ]] && printf '# Changelog\n' >"$repo/plugins/$name/CHANGELOG.md" +} + +# ============================ --check (static) ============================= + +# versioned plugin WITH a changelog -> passes +repo="$(mk_repo)" +mk_plugin "$repo" alpha 1.0.0 yes +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check >/dev/null 2>&1); then ok "versioned plugin with CHANGELOG passes --check"; else fail "versioned+changelog wrongly failed"; fi +rm -rf "$repo" + +# SYNTHETIC MISSING CHANGELOG: versioned plugin, no changelog -> fails +repo="$(mk_repo)" +mk_plugin "$repo" alpha 1.0.0 no +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"MISSING CHANGELOG"*"alpha"* ]]; then ok "versioned plugin without CHANGELOG fails --check (synthetic gap caught)"; else fail "missing-changelog not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# grandfathered plugin without a changelog -> passes +repo="$(mk_repo $'alpha\n')" +mk_plugin "$repo" alpha 1.0.0 no +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check >/dev/null 2>&1); then ok "grandfathered missing-changelog passes --check"; else fail "grandfathered plugin wrongly failed"; fi +rm -rf "$repo" + +# STALE baseline: grandfathered plugin that DOES have a changelog -> fails +repo="$(mk_repo $'alpha\n')" +mk_plugin "$repo" alpha 1.0.0 yes +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check 2>&1)" +rc=$? +stale_count="$(printf '%s\n' "$out" | grep -c 'STALE BASELINE')" +if [[ $rc -ne 0 && "$stale_count" == "1" ]]; then ok "stale baseline entry fails --check (reported exactly once)"; else fail "stale baseline: rc=$rc count=$stale_count out='$out'"; fi +rm -rf "$repo" + +# ============================ --check-bump (diff) ========================= + +# version changed AND changelog has the new version's entry -> passes +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n## [1.1.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "bump + '## [x.y.z]' entry passes --check-bump"; else fail "bump+entry wrongly failed"; fi +rm -rf "$repo" + +# SYNTHETIC MALFORMED ENTRY: version present but as an UNBRACKETED heading +# (## 1.1.0) -> FORMAT error naming the found heading, NOT "UNDOCUMENTED BUMP". +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n## 1.1.0 — 2026-07-20\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"CHANGELOG FORMAT"*"alpha"* && "$out" == *"## 1.1.0"* && "$out" != *"UNDOCUMENTED BUMP"* ]]; then ok "bump + unbracketed heading -> FORMAT error (not UNDOCUMENTED)"; else fail "format-split not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# NEWLY-ADDED ENTRY: base carries an earlier `## [1.0.0]`; the bump ADDS +# `## [1.1.0]` (present at head, absent at base) -> passes. Proves the pass +# path accepts a genuinely new release note. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n## [1.1.0]\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "bump adding a NEW '## [x.y.z]' entry (absent at base) passes --check-bump"; else fail "newly-added entry wrongly failed"; fi +rm -rf "$repo" + +# PRE-EXISTING ENTRY: the `## [1.1.0]` heading already exists in the base +# changelog; the bump only edits plugin.json, adding no new release note -> +# fails as PRE-EXISTING (not UNDOCUMENTED). Proves the gate closes the fail-open +# where a bump reuses a heading that predates the change set. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.1.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"PRE-EXISTING CHANGELOG ENTRY"*"alpha"* && "$out" != *"UNDOCUMENTED BUMP"* ]]; then ok "bump reusing a base-pre-existing '## [x.y.z]' entry fails --check-bump"; else fail "preexisting-entry not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# SEMVER BUILD METADATA: a version like 1.0.1+build.1 must round-trip — the +# fixed-string heading match must not interpret "+" as a regex quantifier, so +# a correctly-added `## [1.0.1+build.1]` entry passes the gate. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.0.1+build.1" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n## [1.0.1+build.1]\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "SemVer build-metadata version with a proper entry passes (no regex leak)"; else fail "build-metadata version wrongly failed"; fi +rm -rf "$repo" + +# PROSE MENTION: the bumped version's `## [1.1.0]` appears only inline and on an +# indented line, never as a column-zero heading -> fails as UNDOCUMENTED. Proves +# the heading match is anchored, so an inline or indented mention neither +# satisfies nor falsely pre-exists the release entry. (The fenced-block case, +# where `## [1.1.0]` sits at column zero inside ```, is covered separately.) +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n## [1.0.0]\n\nNext release will be titled "## [1.1.0]" per convention.\n ## [1.1.0] (example, indented, not a heading)\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "version string in prose/indented example does not satisfy the anchored heading match"; else fail "unanchored-mention not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# FENCED-BLOCK EXAMPLE: the bumped version's `## [1.1.0]` sits at column zero but +# INSIDE a ``` fenced block, never as a real heading -> fails as UNDOCUMENTED. +# Proves the heading match tracks fence state (column-zero alone is not enough). +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +# shellcheck disable=SC2016 # single quotes are deliberate: the backtick fence and \n are literal changelog bytes, not shell expansions +printf '# Changelog\n\nExample of a heading:\n\n```\n## [1.1.0]\n```\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "fenced-code example heading (column zero inside a fence) does not satisfy --check-bump"; else fail "fenced-block heading wrongly satisfied gate: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# MISMATCHED INNER FENCE DELIMITER: a backtick fence contains a ~~~ content line +# (a different delimiter, so NOT a close per CommonMark); a `## [1.1.0]` after it +# is still inside the backtick fence -> fails as UNDOCUMENTED. Proves fence +# tracking closes only on the opening delimiter, not any fence-looking line. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +# shellcheck disable=SC2016 # single quotes are deliberate: the backtick/tilde fences and \n are literal changelog bytes +printf '# Changelog\n\n```\n~~~\n## [1.1.0]\n```\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "mismatched inner fence delimiter (tilde line inside a backtick fence) does not prematurely close"; else fail "fence delimiter mismatch wrongly toggled: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# NON-CLOSING FENCE LINES: a backtick fence contains a ```not-a-close line (text +# after the run) and a four-space-indented ``` line (indented code column, not a +# fence per CommonMark); neither closes, so a `## [1.1.0]` after them is still +# fenced -> fails as UNDOCUMENTED. Proves a close requires a whitespace-only +# suffix and at-most-three-space indentation. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +# shellcheck disable=SC2016 # single quotes are deliberate: fence lines are literal changelog bytes +printf '# Changelog\n\n```\n```not-a-close\n ```\n## [1.1.0]\n```\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "non-closing fence lines (text suffix, four-space indent) do not close a fence"; else fail "non-closing fence line wrongly closed the fence: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# HTML-COMMENT HEADING: the bumped version's heading appears only inside a +# multi-line comment -> not rendered Markdown -> fails as UNDOCUMENTED. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog\n\n\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* && "$out" != *"PRE-EXISTING"* ]]; then ok "heading inside an HTML comment does not satisfy --check-bump"; else fail "HTML-comment heading wrongly satisfied gate: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# SYNTHETIC UNDOCUMENTED BUMP: version changed, changelog edited but WITHOUT an +# entry for the new version (unrelated edit) -> fails. Proves the gate checks +# the version's own entry, not merely that the file was touched. +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +printf '# Changelog\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +printf '# Changelog (typo fix)\n\n## [1.0.0]\n' >"$repo/plugins/alpha/CHANGELOG.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* ]]; then ok "bump + unrelated changelog edit (no new-version entry) fails --check-bump"; else fail "unrelated-edit bump not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# SYNTHETIC UNDOCUMENTED BUMP: version changed, changelog untouched -> fails +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf '{ "name": "alpha", "version": "1.1.0" }\n' >"$repo/plugins/alpha/.claude-plugin/plugin.json" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm bump +out="$(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"UNDOCUMENTED BUMP"*"alpha"* ]]; then ok "bump without changelog fails --check-bump (synthetic undocumented bump caught)"; else fail "undocumented bump not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# version unchanged -> passes (no bump, nothing to document) +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +printf 'unrelated\n' >"$repo/plugins/alpha/README.md" +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm noop +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "unchanged version passes --check-bump"; else fail "unchanged version wrongly failed"; fi +rm -rf "$repo" + +# new plugin absent at base -> --check-bump skips it (static --check owns it) +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +base="$(git -C "$repo" rev-parse HEAD)" +mk_plugin "$repo" beta 1.0.0 no +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm add-beta +if (cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump "$base" >/dev/null 2>&1); then ok "new plugin skipped by --check-bump"; else fail "new plugin wrongly failed --check-bump"; fi +rm -rf "$repo" + +# unresolvable base ref -> exit 2 +repo="$(mk_repo)" +git_init "$repo" +mk_plugin "$repo" alpha 1.0.0 yes +git -C "$repo" add -A >/dev/null && git -C "$repo" commit -qm base +(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump does-not-exist >/dev/null 2>&1) +if [[ $? -eq 2 ]]; then ok "unresolvable base ref -> exit 2"; else fail "bad base ref did not exit 2"; fi +rm -rf "$repo" + +# --check-bump with no base ref -> exit 2 (consistent with other usage errors) +repo="$(mk_repo)" +mk_plugin "$repo" alpha 1.0.0 yes +(cd "$repo" && bash scripts/check-changelog-parity.sh --check-bump >/dev/null 2>&1) +if [[ $? -eq 2 ]]; then ok "--check-bump without base ref -> exit 2"; else fail "missing base ref did not exit 2"; fi +rm -rf "$repo" + +# bad usage -> exit 2 +repo="$(mk_repo)" +(cd "$repo" && bash scripts/check-changelog-parity.sh --nonsense >/dev/null 2>&1) +if [[ $? -eq 2 ]]; then ok "bad mode -> exit 2"; else fail "bad mode did not exit 2"; fi +rm -rf "$repo" + +printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL" +((FAIL == 0)) diff --git a/scripts/check-orphaned-fixtures.sh b/scripts/check-orphaned-fixtures.sh new file mode 100755 index 000000000..b65b397d4 --- /dev/null +++ b/scripts/check-orphaned-fixtures.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash +# Detect eval fixtures that no grader consumes: a file under a skill's +# evals/fixtures/ that no eval case references and no test asserts on. An +# ungraded fixture is dead weight that reads as tested — the "looks-tested" +# trap a merged-PR audit caught (autonomy shipped a large security-binding +# fixture corpus with no eval or test consuming any of it). +# +# scripts/check-orphaned-fixtures.sh discover: list every fixture +# under **/evals/fixtures/ and +# whether a grader consumes it +# scripts/check-orphaned-fixtures.sh --check fail if an un-grandfathered +# fixture is orphaned, or a +# baseline entry is now stale +# +# A fixture at /evals/fixtures/ is CONSUMED when any of: +# * its skill-relative path (evals/fixtures/) appears in the sibling +# grader /evals/evals.json — an eval `files[]` entry, the issue's +# primary consumption path; +# * its basename appears as a whole token in that evals.json (a files[] form +# that spells the path differently still names the file); +# * its basename appears as a whole token in any *.test.* file in the plugin +# — a test that asserts on the fixture is a grader too. +# Consumption matching is deliberately generous (path OR basename, across the +# grader and every test): a false "consumed" only under-reports one orphan, +# whereas a false "orphaned" red-lines a legitimate fixture. The baseline +# absorbs the remaining known-orphan debt. +# +# Scope is **/evals/fixtures/ specifically — eval-grader fixtures, the gate's +# target. Unit-test fixture dirs (…/tests/fixtures, …/scripts/fixtures) are out +# of scope: those are often generated or loaded by directory, not named, and a +# name-matcher cannot honestly grade them. +# +# Existing orphan debt owned by another issue is grandfathered in +# scripts/orphaned-fixtures-baseline.txt (exact fixture paths, one file per +# line — matched by full-string equality, NOT prefix, so a baselined +# `.json` never shadows a new `.json.bak` or `.jsonl` +# sibling). An entry there is a promise the owning issue burns it down; --check +# fails on a stale entry (one that no longer shadows any orphan) so the baseline +# cannot outlive its debt. +# +# Fail-closed: a fixture with no resolvable grader is an orphan unless +# explicitly grandfathered. FIXTURES_BASELINE overrides the baseline path (test +# injection). +set -uo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/.." || exit 2 + +BASELINE="${FIXTURES_BASELINE:-scripts/orphaned-fixtures-baseline.txt}" + +mode="${1:-discover}" +case "$mode" in +discover | --check) ;; +*) + echo "usage: $(basename "$0") [--check]" >&2 + exit 2 + ;; +esac + +# Active baseline entries: strip inline comments and surrounding blank lines. +# Each entry is one exact fixture path, matched by full-string equality below. +entries=() +if [[ -f "$BASELINE" ]]; then + mapfile -t entries < <(sed -E 's/#.*//; s/^[[:space:]]+//; s/[[:space:]]+$//' "$BASELINE" | grep -v '^$') +fi + +is_grandfathered() { + local path="$1" entry + for entry in "${entries[@]}"; do + if [[ "$path" == "$entry" ]]; then + return 0 + fi + done + return 1 +} + +# plugin root that owns a path under plugins//… -> plugins/ +plugin_root_of() { + local path="$1" + # plugins / / rest… -> keep the first two segments + printf 'plugins/%s' "${path#plugins/}" | cut -d/ -f1-2 +} + +# consumed -> 0 if some grader consumes it +consumed() { + local fixture="$1" + local fixtures_dir evals_dir skill_dir evals_json rel base plugin plugin_rel files_values + + fixtures_dir="${fixture%/*}" + # walk up to the nearest 'fixtures' segment (fixtures may nest a subdir) + while [[ "$fixtures_dir" == */* && "${fixtures_dir##*/}" != "fixtures" ]]; do + fixtures_dir="${fixtures_dir%/*}" + done + evals_dir="${fixtures_dir%/fixtures}" # …/evals + skill_dir="${evals_dir%/evals}" # dir that owns evals/ + evals_json="$evals_dir/evals.json" + rel="${fixture#"$skill_dir"/}" # evals/fixtures/ + base="${fixture##*/}" + + # Basename matches are bounded by NON-FILENAME characters on both sides: + # grep -w treats "." as a word boundary, so a referenced valid.json would + # wrongly consume a new unconsumed valid.json.bak sibling. Escape ERE + # metacharacters in the basename, then require the neighbors (if any) to be + # outside [A-Za-z0-9._-] so a longer filename never satisfies the match. + # shellcheck disable=SC2016 # single quotes are deliberate: $ is an ERE metachar being escaped, not a shell expansion + esc_base="$(printf '%s' "$base" | sed -E 's/[][\\.|$(){}?+*^]/\\&/g')" + base_re="(^|[^A-Za-z0-9._-])${esc_base}([^A-Za-z0-9._-]|$)" + + if [[ -f "$evals_json" ]]; then + # Consumption via evals.json is limited to files[] VALUES: a fixture named + # only in a prompt or unrelated metadata string is NOT consumed. Extract every + # files[] string (at any nesting), then match $rel by whole-value equality, or + # the basename bounded within a value (a files[] form that spells the path + # differently still names the file). A shorter $rel is not a substring of a + # longer value under -x. jq failure (missing/invalid json) -> empty -> falls + # through to the test-file check below. + files_values="$(jq -r '.. | objects | .files? // empty | .[]? | select(type == "string")' "$evals_json" 2>/dev/null)" + if printf '%s\n' "$files_values" | grep -qxF -- "$rel"; then + return 0 + fi + if printf '%s\n' "$files_values" | grep -qE -- "$base_re"; then + return 0 + fi + fi + + # Test-file consumption is scoped: a bounded-basename match counts only inside + # the OWNING skill, so same-named fixtures in sibling skills are never + # conflated. A test elsewhere in the plugin must name the fixture by its + # plugin-relative path (fixed string), which is unambiguous across skills. + while IFS= read -r -d '' test_file; do + if grep -qE "$base_re" "$test_file"; then + return 0 + fi + done < <(find "$skill_dir" -type f -name '*.test.*' -print0 2>/dev/null) + + plugin="$(plugin_root_of "$fixture")" + plugin_rel="${fixture#"$plugin"/}" + while IFS= read -r -d '' test_file; do + case "$test_file" in "$skill_dir"/*) continue ;; *) ;; esac + if grep -qF -- "$plugin_rel" "$test_file"; then + return 0 + fi + done < <(find "$plugin" -type f -name '*.test.*' -print0 2>/dev/null) + + return 1 +} + +# Collect every fixture under a **/evals/fixtures/ directory. +fixtures=() +while IFS= read -r -d '' dir; do + while IFS= read -r -d '' f; do + fixtures+=("$f") + done < <(find "$dir" -type f -print0) +done < <(find plugins -type d -path '*/evals/fixtures' -print0 2>/dev/null) + +if ((${#fixtures[@]} > 0)); then + mapfile -t -d '' fixtures < <(printf '%s\0' "${fixtures[@]}" | sort -z) +fi + +# Track which baseline entries actually shadow an orphan, to flag stale ones. +declare -A entry_used +matched_baseline() { + local path="$1" entry + for entry in "${entries[@]}"; do + if [[ "$path" == "$entry" ]]; then + printf '%s' "$entry" + return 0 + fi + done + return 1 +} + +orphans=0 +if [[ "$mode" == "discover" ]]; then + for f in "${fixtures[@]}"; do + if consumed "$f"; then + printf '%-14s %s\n' "CONSUMED" "$f" + elif is_grandfathered "$f"; then + printf '%-14s %s\n' "GRANDFATHERED" "$f" + else + printf '%-14s %s\n' "ORPHAN" "$f" + fi + done + exit 0 +fi + +# --check mode +for f in "${fixtures[@]}"; do + if consumed "$f"; then + continue + fi + if p="$(matched_baseline "$f")"; then + entry_used["$p"]=1 + continue + fi + echo "ORPHANED FIXTURE: $f is under evals/fixtures/ but no eval case references it and no test asserts on it." >&2 + echo " Reference it from a grader (an eval files[] entry or a test), delete it, or grandfather it in $BASELINE with the owning issue." >&2 + orphans=$((orphans + 1)) +done + +stale=0 +for entry in "${entries[@]}"; do + if [[ -z "${entry_used[$entry]:-}" ]]; then + echo "STALE BASELINE: '$entry' in $BASELINE no longer shadows any orphaned fixture — remove it." >&2 + stale=$((stale + 1)) + fi +done + +if ((orphans > 0 || stale > 0)); then + exit 1 +fi +echo "No orphaned eval fixtures (every file under **/evals/fixtures/ is consumed by a grader or grandfathered)." diff --git a/scripts/check-orphaned-fixtures.test.sh b/scripts/check-orphaned-fixtures.test.sh new file mode 100755 index 000000000..a8952534b --- /dev/null +++ b/scripts/check-orphaned-fixtures.test.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Unit tests for check-orphaned-fixtures.sh. Each scenario builds a throwaway +# plugins/ tree, runs the detector, and asserts the outcome. The negative cases +# (a fixture no grader consumes) are the synthetic proof the gate catches the +# "looks-tested but graded by nothing" class it exists for. +set -uo pipefail + +SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="$SELF_DIR/check-orphaned-fixtures.sh" + +PASS=0 +FAIL=0 +fail() { + echo "FAIL: $*" >&2 + FAIL=$((FAIL + 1)) +} +ok() { + echo "ok: $*" + PASS=$((PASS + 1)) +} + +# mk_repo : throwaway repo with the detector installed and a +# baseline file at scripts/orphaned-fixtures-baseline.txt. +mk_repo() { + local dir + dir="$(mktemp -d)" + mkdir -p "$dir/scripts" + cp "$SCRIPT" "$dir/scripts/check-orphaned-fixtures.sh" + printf '%s' "${1:-}" >"$dir/scripts/orphaned-fixtures-baseline.txt" + printf '%s' "$dir" +} + +# seed a skill's evals.json with an optional files[] entry +seed_skill() { + local repo="$1" skill="$2" files_entry="$3" + mkdir -p "$repo/$skill/evals/fixtures" + cat >"$repo/$skill/evals/evals.json" </dev/null) +} + +# --- consumed via an eval files[] entry -> not an orphan -------------------- +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '"evals/fixtures/used.md"' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/used.md" +if run_check "$repo" >/dev/null; then ok "files[]-referenced fixture passes --check"; else fail "files[]-referenced fixture wrongly flagged"; fi +rm -rf "$repo" + +# --- consumed via a test file (basename) -> not an orphan ------------------ +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/by-test.md" +mkdir -p "$repo/plugins/p/skills/s/scripts" +printf 'assert_on fixtures/by-test.md\n' >"$repo/plugins/p/skills/s/scripts/thing.test.sh" +if run_check "$repo" >/dev/null; then ok "test-asserted fixture passes --check"; else fail "test-asserted fixture wrongly flagged"; fi +rm -rf "$repo" + +# --- REFERENCE-NAME SUFFIX SIBLING: a referenced valid.json must NOT consume +# a new valid.json.bak (grep -w treated "." as a word boundary; the bounded +# match rejects filename-character neighbors) -> .bak sibling is an orphan --- +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '"evals/fixtures/valid.json"' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json" +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json.bak" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"valid.json.bak"* && "$out" != *"ORPHANED FIXTURE"*"fixtures/valid.json is"* ]]; then ok "suffix sibling of a referenced fixture red-lines (bounded basename match)"; else fail "reference-name suffix sibling not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# --- files[] PATH match is EXACT, not a whole-file substring: an eval that +# references the longer evals/fixtures/valid.json.bak must NOT consume a new +# evals/fixtures/valid.json sibling (shorter $rel is a substring of the longer +# files[] value) -> valid.json red-lines, valid.json.bak stays consumed ------- +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '"evals/fixtures/valid.json.bak"' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json.bak" +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"fixtures/valid.json is"* && "$out" != *"valid.json.bak is"* ]]; then ok "files[] path match is exact (substring of a longer referenced path does not consume)"; else fail "files[] substring path match not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# --- SIBLING-SKILL BASENAME CONFLATION: skill a's test naming shared.md must +# NOT consume skill b's same-named fixture (basename matches count only inside +# the owning skill; cross-skill consumption needs the plugin-relative path) ---- +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/a" '' +seed_skill "$repo" "plugins/p/skills/b" '' +printf 'x\n' >"$repo/plugins/p/skills/a/evals/fixtures/shared.md" +printf 'x\n' >"$repo/plugins/p/skills/b/evals/fixtures/shared.md" +mkdir -p "$repo/plugins/p/skills/a/scripts" +printf 'assert_on fixtures/shared.md\n' >"$repo/plugins/p/skills/a/scripts/thing.test.sh" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"skills/b/evals/fixtures/shared.md"* && "$out" != *"skills/a/evals/fixtures/shared.md is"* ]]; then ok "sibling skill's same-named fixture is not conflated (test basename scoped to owning skill)"; else fail "sibling-skill basename conflation not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# --- CROSS-SKILL CONSUMPTION BY PLUGIN-RELATIVE PATH: a plugin-root test naming +# skills/b/evals/fixtures/by-plugin-test.md consumes it ------------------------ +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/b" '' +printf 'x\n' >"$repo/plugins/p/skills/b/evals/fixtures/by-plugin-test.md" +mkdir -p "$repo/plugins/p/tools" +printf 'assert_on skills/b/evals/fixtures/by-plugin-test.md\n' >"$repo/plugins/p/tools/suite.test.sh" +if run_check "$repo" >/dev/null; then ok "plugin-root test consuming by plugin-relative path passes --check"; else fail "plugin-relative path consumption wrongly flagged"; fi +rm -rf "$repo" + +# --- evals.json consumption is limited to files[] VALUES: a fixture named only +# in a prompt/metadata string, with an empty files[], is NOT consumed -> orphan - +repo="$(mk_repo)" +mkdir -p "$repo/plugins/p/skills/s/evals/fixtures" +cat >"$repo/plugins/p/skills/s/evals/evals.json" <<'JSON' +{ "evals": [ { "id": 1, "prompt": "open evals/fixtures/prose-only.md and grade the summary", "files": [] } ] } +JSON +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/prose-only.md" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"prose-only.md"* ]]; then ok "fixture named only in an eval prompt (empty files[]) is an orphan (consumption scoped to files[])"; else fail "prose-mention wrongly consumed: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# --- SYNTHETIC ORPHAN: consumed by nothing -> --check fails ---------------- +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/orphan.md" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"ORPHANED FIXTURE"*"orphan.md"* ]]; then ok "un-consumed fixture fails --check (synthetic orphan caught)"; else fail "synthetic orphan not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# --- orphan grandfathered by an exact baseline path -> passes -------------- +repo="$(mk_repo $'plugins/p/skills/s/evals/fixtures/orphan.md\n')" +seed_skill "$repo" "plugins/p/skills/s" '' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/orphan.md" +if run_check "$repo" >/dev/null; then ok "grandfathered orphan (exact path) passes --check"; else fail "grandfathered orphan wrongly failed"; fi +rm -rf "$repo" + +# --- baseline entry is EXACT, not a prefix: a .bak/.jsonl sibling of a +# baselined file is NOT grandfathered and red-lines -------------------- +repo="$(mk_repo $'plugins/p/skills/s/evals/fixtures/valid.json\n')" +seed_skill "$repo" "plugins/p/skills/s" '' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json" +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.json.bak" +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/valid.jsonl" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"valid.json.bak"* && "$out" == *"valid.jsonl"* ]]; then ok "prefix-sibling of a baselined path red-lines (exact-match, no grandfather leak)"; else fail "baseline prefix-leak not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# --- STALE baseline entry (shadows no orphan) -> fails --------------------- +repo="$(mk_repo $'plugins/p/skills/s/evals/fixtures/gone.md\n')" +seed_skill "$repo" "plugins/p/skills/s" '"evals/fixtures/used.md"' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/used.md" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --check 2>&1)" +rc=$? +if [[ $rc -ne 0 && "$out" == *"STALE BASELINE"* ]]; then ok "stale baseline entry fails --check"; else fail "stale baseline not caught: rc=$rc out='$out'"; fi +rm -rf "$repo" + +# --- discover mode labels every fixture ------------------------------------ +repo="$(mk_repo)" +seed_skill "$repo" "plugins/p/skills/s" '"evals/fixtures/used.md"' +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/used.md" +printf 'x\n' >"$repo/plugins/p/skills/s/evals/fixtures/orphan.md" +out="$(cd "$repo" && bash scripts/check-orphaned-fixtures.sh discover 2>/dev/null)" +if [[ "$out" == *"CONSUMED"*"used.md"* && "$out" == *"ORPHAN"*"orphan.md"* ]]; then ok "discover labels CONSUMED and ORPHAN"; else fail "discover mislabels: '$out'"; fi +rm -rf "$repo" + +# --- bad usage -> exit 2 --------------------------------------------------- +repo="$(mk_repo)" +(cd "$repo" && bash scripts/check-orphaned-fixtures.sh --nonsense >/dev/null 2>&1) +if [[ $? -eq 2 ]]; then ok "bad mode -> exit 2"; else fail "bad mode did not exit 2"; fi +rm -rf "$repo" + +printf '\nPASS=%d FAIL=%d\n' "$PASS" "$FAIL" +((FAIL == 0)) diff --git a/scripts/orphaned-fixtures-baseline.txt b/scripts/orphaned-fixtures-baseline.txt new file mode 100644 index 000000000..cb8b38bad --- /dev/null +++ b/scripts/orphaned-fixtures-baseline.txt @@ -0,0 +1,208 @@ +# Grandfathered orphaned eval fixtures — existing debt this gate must not +# red-line, each owned by (or awaiting) a burn-down issue. Each line is one +# exact fixture path, matched by full-string EQUALITY (not prefix): a line +# exempts only that file from check-orphaned-fixtures.sh --check. This is a +# snapshot of today's known orphans, so a NEW orphan — whether added under an +# already-listed directory or as a `.json.bak` / `.jsonl` sibling +# of a listed file — matches no line and is red-lined rather than grandfathered. +# +# An entry here is a standing promise the debt is tracked, not a place to hide +# new orphans: --check fails on a STALE entry (one that no longer shadows any +# orphan), so a line cannot outlive its debt. Delete the line when the owning +# issue lands the fix. Do NOT add a plugin's fixtures here to dodge the gate on +# NEW work — reference them from a grader instead. + +# autonomy setup skill: the security-binding golden suite (referenced by no +# eval files[] entry and graded by no test) plus the otlp-demo sample corpus, +# enumerated file-by-file below. The security-binding subset is tracked by +# issue #662 (attach expected-verdict blocks and wire them into the grader, or +# demote them to documentation exemplars out of evals/fixtures/); otlp-demo is +# the same ungraded class in the same skill. autonomy is WP-lane-owned — the fix +# is coordinated there, not here. +plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/pipeline.jsonl +plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/session-metrics.jsonl +plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/session-spans.jsonl +plugins/autonomy/skills/setup/evals/fixtures/otlp-demo/trigger.jsonl +plugins/autonomy/skills/setup/evals/fixtures/security-binding/admission-weakened-no-justification.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ambiguous-runtime-markers.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/as112-v6-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/c2-auto-unratified.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/c3-blocking-unratified.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/c4-auto-merge.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/equal-specificity-tie.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-ack-invalid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-ack-loop.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-push-without-channel.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-runner-route-missing.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-severity-bad-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-severity-tracker-only-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/escalation-severity-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/evidence-contrary.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/evidence-stale.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/hosted-ephemeral-l3-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/human-gated-only-no-l2.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/l3-ratified-class-container.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/marker-prototype-key.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/missing-l2-autonomous.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/missing-probe-evidence.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/missing-substrate-class.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/multi-host-egress-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-6to4-relay.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-6to4-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-bare-metadata.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-benchmarking-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-bogus-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-bogus-metadata.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-cred-outer-missing.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-cred-outer-nonzero.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-credpath-code-count.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-credpath-partial-zero.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-discard-only.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-doc-3fff.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-docnet-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-dot-traversal.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-dummy-prefix.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-encoded-loopback.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-etc-descendant-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-expanded-loopback.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-expansion-ephemeral.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-expansion-tail-mismatch.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-generic-env-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-generic-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-home-descendant-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-host-home-mount.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-imds-http-error.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invalid-tld.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invented-env-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invented-home-user.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-invented-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-ipv4-compat.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-known-hosts.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-l3-container.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-level-mismatch.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-local-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-malformed-dns.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-port.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-route-boundary.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-route.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-metadata-scheme.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-host-expanded.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-outer-exit.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-probed-at.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-target.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-missing-transport.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-multi-host-private.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-nat64-local.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-nat64-private-embed.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-outer-egress-failed.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-protocol-assignment.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-ratified-class-mismatch.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-relative-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-root-escape.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-site-local-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-srv6-sid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-ssh-dir.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-teredo.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-token-substring.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-transport-cross-type.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-unanchored-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-unanchored-envvar.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-unc-etc.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-uri-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-evidence-zero-exit.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-6to4-relay.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-6to4-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-as112-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-bare-metadata.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-benchmarking-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-bogus-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-bogus-metadata.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-cred-outer-missing.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-cred-outer-nonzero.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-credpath-code-count.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-credpath-partial-zero.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-discard-only.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-doc-3fff.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-docnet.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-dot-traversal.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-dummy-prefix.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-encoded-loopback.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-etc-descendant-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-expanded-loopback.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-expansion-ephemeral.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-expansion-tail-mismatch.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-generic-env-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-generic-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-home-descendant-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-host-home-mount.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-id-rsa.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-imds-http-error.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invalid-tld.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invented-env-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invented-home-user.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-invented-token.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-ipv4-compat.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-known-hosts.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-localhost.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-malformed-dns.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-port.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-route-boundary.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-route.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-metadata-scheme.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-multi-host-private.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-multi-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-nat64-local.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-nat64-private-embed.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-host-expanded.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-outer-exit.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-probed-at.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-no-transport.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-outer-egress-failed.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-protocol-assignment.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-relative-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-site-local-v6.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-srv6-sid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-ssh-dir.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-teredo.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-token-substring.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-transport-cross-type.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-unanchored-credpath.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-unanchored-envvar.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-unc-etc.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-uri-host.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2-zero-exit.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l2.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l3-container.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l3-hosted.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-a-l3.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/probe-transcripts/ci-pool-b-l2.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/promoted-c2.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/promotion-nonpromotable.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-calendar-invalid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-non-iso.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-offset-invalid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ratified-at-submilli.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/same-surface-level-ambiguity.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/ssh-id-rsa-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-classification-bare-string.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-classification-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-duplicate-producer-identity.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-empty-source-surface.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-identity-grammar.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-missing-producer-identity.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-missing-run-link-prefix.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-shared-run-link-prefix-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/temporal-shared-source-surface.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/two-level-surface-valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/unverifiable-probe-evidence.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/valid.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/vendor-hosted-auto.json +plugins/autonomy/skills/setup/evals/fixtures/security-binding/verification-below-floor.json + +# knowledge youtube-digest skill: this data file lives under evals/fixtures/ but +# is named only from SKILL.md / vendor TUNING.md prose — no eval files[] entry +# and no test consume it. Surfaced by this gate's initial discover run; tracked +# by issue #688 (wire it into a grader, or move it out of evals/fixtures/ if it +# is reference data rather than a fixture). +plugins/knowledge/skills/youtube-digest/evals/fixtures/variation-matrix-backlog.json