diff --git a/.changes/harness-ops/test-8206-plugins-audit-native-cases-7c2e9b51.md b/.changes/harness-ops/test-8206-plugins-audit-native-cases-7c2e9b51.md new file mode 100644 index 00000000000..8ad43e5d746 --- /dev/null +++ b/.changes/harness-ops/test-8206-plugins-audit-native-cases-7c2e9b51.md @@ -0,0 +1,5 @@ +--- +bump: none +--- + +No release needed (evals): the four `plugins-audit` eval cases move from the skill's legacy `evals.json` to `claude plugin eval` cases under `evals/`. Three of them get a scaffold that seeds a fixture fleet in the run's home (refs #9966, #8206) diff --git a/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/case.yaml b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/case.yaml new file mode 100644 index 00000000000..c6774d07a7c --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/case.yaml @@ -0,0 +1,25 @@ +schema_version: "1.1" +name: 'plugins-audit-audit-issues-zero-mutating-calls' +graders: + - name: 'expectation-1' + type: 'llm' + criteria: 'The response issues zero `claude plugin install|update|uninstall` or `claude plugin marketplace update` invocations' + focus: 'trace' + - name: 'expectation-2' + type: 'llm' + criteria: 'The response still computes state from the read-only fleet-state.sh reads, which sync-run.sh --audit takes' + focus: 'trace' + - name: 'expectation-3' + type: 'llm' + criteria: 'The response still takes Step 1''s pre-refresh fleet-state.sh snapshot before declining the refresh: it does not treat the whole of Step 1 as skipped, and the catalog-regression check still has its pre-refresh to pre interval available' + focus: 'trace' + - name: 'expectation-4' + type: 'llm' + criteria: 'The response frames its output as a prediction of what `sync` (and, where relevant, `converge`) would do, not as work already performed' + focus: 'trace' + - name: 'expectation-5' + type: 'llm' + criteria: 'If the response mentions enabledPlugins key order, it uses normalize-enabled-plugins.sh --check (or equivalent prediction) and never writes ~/.claude/settings.json or a committed project settings file' + focus: 'trace' +context: + scaffold_script: scaffold.sh diff --git a/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/prompt.md b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/prompt.md new file mode 100644 index 00000000000..8eecc9fe632 --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/prompt.md @@ -0,0 +1,8 @@ +--- +description: "The scaffold seeds a fixture fleet in the run's home (a marketplace, its catalog, install records and settings) so the default marketplace resolves. Needs --scaffold --allow-tools Bash, with the real node first on PATH: a mise shim on PATH fails under the run's temporary home." +tags: ['plugins-audit'] +allowed_tools: [Read, Glob, Grep, Skill, Bash] +expected_outcome: 'audit runs the same algorithm as sync but every mutating CLI call becomes a prediction: no `claude plugin install|update|uninstall|marketplace update` command is ever run, and normalize-enabled-plugins.sh is invoked with --check rather than written. The reads those calls sit beside still run. Step 1''s pre-refresh fleet-state.sh snapshot is taken (into the run''s throwaway scratch directory) so the catalog-regression check keeps its first interval, and only the `claude plugin marketplace update` beside it is replaced by a prediction. The response calls fleet-state.sh (read-only) and reports what sync/converge would do without doing it.' +--- + +/harness-ops:plugins-audit diff --git a/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-1.json b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-1.json new file mode 100644 index 00000000000..e5ecaa7239c --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-1.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "audit runs the same algorithm as sync but every mutating CLI call becomes a prediction: no `claude plugin install|update|uninstall|marketplace update` command is ever run, and normalize-enabled-plugins.sh is invoked with --check rather than written. The reads those calls sit beside still run. Step 1's pre-refresh fleet-state.sh snapshot is taken (into the run's throwaway scratch directory) so the catalog-regression check keeps its first interval, and only the `claude plugin marketplace update` beside it is replaced by a prediction. The response calls fleet-state.sh (read-only) and reports what sync/converge would do without doing it.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-2.json b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-2.json new file mode 100644 index 00000000000..e5ecaa7239c --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-2.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "audit runs the same algorithm as sync but every mutating CLI call becomes a prediction: no `claude plugin install|update|uninstall|marketplace update` command is ever run, and normalize-enabled-plugins.sh is invoked with --check rather than written. The reads those calls sit beside still run. Step 1's pre-refresh fleet-state.sh snapshot is taken (into the run's throwaway scratch directory) so the catalog-regression check keeps its first interval, and only the `claude plugin marketplace update` beside it is replaced by a prediction. The response calls fleet-state.sh (read-only) and reports what sync/converge would do without doing it.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-3.json b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-3.json new file mode 100644 index 00000000000..e5ecaa7239c --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-3.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "audit runs the same algorithm as sync but every mutating CLI call becomes a prediction: no `claude plugin install|update|uninstall|marketplace update` command is ever run, and normalize-enabled-plugins.sh is invoked with --check rather than written. The reads those calls sit beside still run. Step 1's pre-refresh fleet-state.sh snapshot is taken (into the run's throwaway scratch directory) so the catalog-regression check keeps its first interval, and only the `claude plugin marketplace update` beside it is replaced by a prediction. The response calls fleet-state.sh (read-only) and reports what sync/converge would do without doing it.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-4.json b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-4.json new file mode 100644 index 00000000000..e5ecaa7239c --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-4.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "audit runs the same algorithm as sync but every mutating CLI call becomes a prediction: no `claude plugin install|update|uninstall|marketplace update` command is ever run, and normalize-enabled-plugins.sh is invoked with --check rather than written. The reads those calls sit beside still run. Step 1's pre-refresh fleet-state.sh snapshot is taken (into the run's throwaway scratch directory) so the catalog-regression check keeps its first interval, and only the `claude plugin marketplace update` beside it is replaced by a prediction. The response calls fleet-state.sh (read-only) and reports what sync/converge would do without doing it.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-5.json b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-5.json new file mode 100644 index 00000000000..e5ecaa7239c --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/samples/expectation-5.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "audit runs the same algorithm as sync but every mutating CLI call becomes a prediction: no `claude plugin install|update|uninstall|marketplace update` command is ever run, and normalize-enabled-plugins.sh is invoked with --check rather than written. The reads those calls sit beside still run. Step 1's pre-refresh fleet-state.sh snapshot is taken (into the run's throwaway scratch directory) so the catalog-regression check keeps its first interval, and only the `claude plugin marketplace update` beside it is replaced by a prediction. The response calls fleet-state.sh (read-only) and reports what sync/converge would do without doing it.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/scaffold.sh b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/scaffold.sh new file mode 100755 index 00000000000..b47b1b32ecb --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-audit-issues-zero-mutating-calls/scaffold.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Seeds a fixture fleet in the run's home: one marketplace, one plugin behind its catalog, one current, one new. +# The runner sets HOME to the run's temporary home for a scaffold script +# (https://code.claude.com/docs/en/plugin-evals, "Set up fixtures and mocks"). +# The guard refuses the account's real home and never overwrites a file. +set -euo pipefail + +refuse() { + echo "refusing: $1" >&2 + exit 1 +} + +[[ -n "${HOME:-}" && -d "$HOME" ]] || refuse "HOME is unset or not a directory" +user=$(id -un) +if command -v getent >/dev/null 2>&1; then + real_home=$(getent passwd "$user" | cut -d: -f6) +elif command -v dscl >/dev/null 2>&1; then + real_home=$(dscl . -read "/Users/$user" NFSHomeDirectory | awk '{print $2}') +else + refuse "cannot look up the account's real home to compare against" +fi +[[ -n "$real_home" ]] || refuse "the account's real home did not resolve" +[[ "$(cd "$HOME" && pwd -P)" != "$(cd "$real_home" && pwd -P)" ]] || refuse "HOME is the account's real home: $HOME" + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +p="$HOME/.claude/plugins" +m="$p/marketplaces/fixture-market" +targets=("$p/known_marketplaces.json" "$p/installed_plugins.json" "$HOME/.claude/settings.json" + "$HOME/.claude/settings.local.json" "$PWD/.claude/settings.local.json" "$PWD/.claude/settings.json") +for f in "${targets[@]}" "$m"; do [[ ! -e "$f" ]] || refuse "$f already exists"; done + +mkdir -p "$m/.claude-plugin" "$p/cache" "$PWD/.claude" +for spec in alpha:2.0.0 beta:1.0.0 gamma:1.0.0 harness-ops:0.0.0; do + n=${spec%%:*} + v=${spec##*:} + mkdir -p "$m/$n/.claude-plugin" + printf '{"name":"%s","version":"%s"}\n' "$n" "$v" >"$m/$n/.claude-plugin/plugin.json" +done +cat >"$m/.claude-plugin/marketplace.json" <<'JSON' +{"name":"fixture-market","owner":{"name":"fixture"},"plugins":[ +{"name":"alpha","source":"./alpha"}, +{"name":"beta","source":"./beta"}, +{"name":"gamma","source":"./gamma"}, +{"name":"harness-ops","source":"./harness-ops"}]} +JSON +cat >"$p/known_marketplaces.json" <"$p/installed_plugins.json" <"$HOME/.claude/settings.json" +for f in "$HOME/.claude/settings.local.json" "$PWD/.claude/settings.local.json" "$PWD/.claude/settings.json"; do printf '{}\n' >"$f"; done diff --git a/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/case.yaml b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/case.yaml new file mode 100644 index 00000000000..03197c3ba95 --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/case.yaml @@ -0,0 +1,21 @@ +schema_version: "1.1" +name: 'plugins-audit-divergence-report-collapses-bulk-names-conflicts' +graders: + - name: 'expectation-1' + type: 'llm' + criteria: 'The 29 actionable divergences are collapsed into a single bulk-count line, not enumerated one by one; per-project-path sub-lines under it are allowed, each naming at most 3 ids, followed by `+K more` when more ids remain' + focus: 'trace' + - name: 'expectation-2' + type: 'llm' + criteria: 'The 2 same-version (versionsMatch: true) multi-scope installs are excluded from the actionable divergence count' + focus: 'trace' + - name: 'expectation-3' + type: 'llm' + criteria: 'The delisted_settings_only plugin id gets an individual per-row detail line under Action needed, separate from the bulk divergence line' + focus: 'trace' + - name: 'expectation-4' + type: 'llm' + criteria: 'The response never claims to detect or report an enable-state mismatch (true in one scope, false in another) — fleet-state.sh only exposes the merged effective value, not each scope''s raw enabledPlugins map' + focus: 'trace' +context: + scaffold_script: scaffold.sh diff --git a/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/prompt.md b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/prompt.md new file mode 100644 index 00000000000..f655f306a24 --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/prompt.md @@ -0,0 +1,8 @@ +--- +description: "The scaffold seeds a fixture fleet in the run's home (a marketplace, its catalog, install records and settings) so the default marketplace resolves. Needs --scaffold --allow-tools Bash, with the real node first on PATH: a mise shim on PATH fails under the run's temporary home." +tags: ['plugins-audit'] +allowed_tools: [Read, Glob, Grep, Skill, Bash] +expected_outcome: 'The report''s Divergences line collapses the 29 actionable (version-behind) entries into one bulk count and names `/harness-ops:plugins converge` as the fix — it does NOT list all 29 individually (at most one sub-line per project path, each naming that path''s record count and up to 3 ids, then `+K more` when more ids remain), and it does NOT count the 2 same-version multi-scope installs as actionable (versionsMatch: true is benign). The one delisted_settings_only plugin id gets its own per-row detail line under Action needed as a genuine conflict, distinct from the bulk divergence count.' +--- + +/harness-ops:plugins-audit — I have 29 plugins with an actionable version divergence between project and user scope, plus 2 more that are multi-scope but pinned to the identical version, and one plugin id that fleet-state.sh reports under delisted_settings_only (true in enabledPlugins, no install record, and no matching catalog entry). diff --git a/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-1.json b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-1.json new file mode 100644 index 00000000000..451330a250e --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-1.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The report's Divergences line collapses the 29 actionable (version-behind) entries into one bulk count and names `/harness-ops:plugins converge` as the fix — it does NOT list all 29 individually (at most one sub-line per project path, each naming that path's record count and up to 3 ids, then `+K more` when more ids remain), and it does NOT count the 2 same-version multi-scope installs as actionable (versionsMatch: true is benign). The one delisted_settings_only plugin id gets its own per-row detail line under Action needed as a genuine conflict, distinct from the bulk divergence count.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-2.json b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-2.json new file mode 100644 index 00000000000..451330a250e --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-2.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The report's Divergences line collapses the 29 actionable (version-behind) entries into one bulk count and names `/harness-ops:plugins converge` as the fix — it does NOT list all 29 individually (at most one sub-line per project path, each naming that path's record count and up to 3 ids, then `+K more` when more ids remain), and it does NOT count the 2 same-version multi-scope installs as actionable (versionsMatch: true is benign). The one delisted_settings_only plugin id gets its own per-row detail line under Action needed as a genuine conflict, distinct from the bulk divergence count.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-3.json b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-3.json new file mode 100644 index 00000000000..451330a250e --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-3.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The report's Divergences line collapses the 29 actionable (version-behind) entries into one bulk count and names `/harness-ops:plugins converge` as the fix — it does NOT list all 29 individually (at most one sub-line per project path, each naming that path's record count and up to 3 ids, then `+K more` when more ids remain), and it does NOT count the 2 same-version multi-scope installs as actionable (versionsMatch: true is benign). The one delisted_settings_only plugin id gets its own per-row detail line under Action needed as a genuine conflict, distinct from the bulk divergence count.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-4.json b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-4.json new file mode 100644 index 00000000000..451330a250e --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/samples/expectation-4.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The report's Divergences line collapses the 29 actionable (version-behind) entries into one bulk count and names `/harness-ops:plugins converge` as the fix — it does NOT list all 29 individually (at most one sub-line per project path, each naming that path's record count and up to 3 ids, then `+K more` when more ids remain), and it does NOT count the 2 same-version multi-scope installs as actionable (versionsMatch: true is benign). The one delisted_settings_only plugin id gets its own per-row detail line under Action needed as a genuine conflict, distinct from the bulk divergence count.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/scaffold.sh b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/scaffold.sh new file mode 100755 index 00000000000..b7b4b53f4f3 --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-divergence-report-collapses-bulk-names-conflicts/scaffold.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Seeds the fleet the divergence case names: 29 plugins whose project and user scopes differ, 2 same-version multi-scope, 1 settings-only delisted id. +# The runner sets HOME to the run's temporary home for a scaffold script +# (https://code.claude.com/docs/en/plugin-evals, "Set up fixtures and mocks"). +# The guard refuses the account's real home and never overwrites a file. +set -euo pipefail + +refuse() { + echo "refusing: $1" >&2 + exit 1 +} + +[[ -n "${HOME:-}" && -d "$HOME" ]] || refuse "HOME is unset or not a directory" +user=$(id -un) +if command -v getent >/dev/null 2>&1; then + real_home=$(getent passwd "$user" | cut -d: -f6) +elif command -v dscl >/dev/null 2>&1; then + real_home=$(dscl . -read "/Users/$user" NFSHomeDirectory | awk '{print $2}') +else + refuse "cannot look up the account's real home to compare against" +fi +[[ -n "$real_home" ]] || refuse "the account's real home did not resolve" +[[ "$(cd "$HOME" && pwd -P)" != "$(cd "$real_home" && pwd -P)" ]] || refuse "HOME is the account's real home: $HOME" + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +p="$HOME/.claude/plugins" +m="$p/marketplaces/fixture-market" +other="$HOME/other-project" +targets=("$p/known_marketplaces.json" "$p/installed_plugins.json" "$HOME/.claude/settings.json" + "$HOME/.claude/settings.local.json" "$PWD/.claude/settings.local.json" "$PWD/.claude/settings.json") +for f in "${targets[@]}" "$m" "$other"; do [[ ! -e "$f" ]] || refuse "$f already exists"; done + +mkdir -p "$m/.claude-plugin" "$p/cache" "$PWD/.claude" "$other/.claude" +git -C "$PWD" init -q 2>/dev/null || true +git -C "$other" init -q +installed="" +catalog="" +enabled="" +rec() { # id scope version [projectPath] + local extra="" + [[ -z "${4:-}" ]] || extra=",\"projectPath\":\"$4\"" + printf '{"scope":"%s"%s,"installPath":"%s/cache/fixture-market/%s/%s","version":"%s","installedAt":"2026-10-01T00:00:00.000Z","lastUpdated":"2026-10-01T00:00:00.000Z"}' "$2" "$extra" "$p" "${1%@*}" "$3" "$3" +} +add() { # name user-version project-version + local n=$1 id="$1@fixture-market" + mkdir -p "$m/$n/.claude-plugin" + printf '{"name":"%s","version":"%s"}\n' "$n" "${4:-$2}" >"$m/$n/.claude-plugin/plugin.json" + catalog+="${catalog:+,}{\"name\":\"$n\",\"source\":\"./$n\"}" + installed+="${installed:+,}\"$id\":[$(rec "$id" user "$2"),$(rec "$id" project "$3" "$other")]" + enabled+="${enabled:+,}\"$id\":true" +} +for i in $(seq -w 1 29); do add "skew$i" 1.0.0 2.0.0; done +add same1 1.0.0 1.0.0 +add same2 1.0.0 1.0.0 +mkdir -p "$m/harness-ops/.claude-plugin" +printf '{"name":"harness-ops","version":"0.0.0"}\n' >"$m/harness-ops/.claude-plugin/plugin.json" +catalog+=',{"name":"harness-ops","source":"./harness-ops"}' +enabled+=",\"gone@fixture-market\":true,\"harness-ops@fixture-market\":true" +printf '{"name":"fixture-market","owner":{"name":"fixture"},"plugins":[%s]}\n' "$catalog" >"$m/.claude-plugin/marketplace.json" +printf '{"fixture-market":{"source":{"source":"github","repo":"fixture/market"},"installLocation":"%s","lastUpdated":"2026-11-01T00:00:00.000Z","autoUpdate":false}}\n' "$m" >"$p/known_marketplaces.json" +installed+=",\"harness-ops@fixture-market\":[{\"scope\":\"user\",\"installPath\":\"$root\",\"version\":\"0.0.0\",\"installedAt\":\"2026-10-01T00:00:00.000Z\",\"lastUpdated\":\"2026-10-01T00:00:00.000Z\"}]" +printf '{"version":2,"plugins":{%s}}\n' "$installed" >"$p/installed_plugins.json" +printf '{"enabledPlugins":{%s}}\n' "$enabled" >"$HOME/.claude/settings.json" +for f in "$HOME/.claude/settings.local.json" "$PWD/.claude/settings.local.json" "$PWD/.claude/settings.json"; do printf '{}\n' >"$f"; done diff --git a/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/case.yaml b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/case.yaml new file mode 100644 index 00000000000..db8c78f04bb --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/case.yaml @@ -0,0 +1,21 @@ +schema_version: "1.1" +name: 'plugins-audit-model-starts-audit-not-sync' +graders: + - name: 'expectation-1' + type: 'llm' + criteria: 'The script call carries --audit and none of --journal-root, --pull-source or --only-install' + focus: 'trace' + - name: 'expectation-2' + type: 'llm' + criteria: 'The response does not run or start /harness-ops:plugins sync or converge itself; it names sync as the user''s next step' + focus: 'trace' + - name: 'expectation-3' + type: 'llm' + criteria: 'A prediction of zero updates is framed as a lower bound against the unrefreshed catalog, not as the fleet being current' + focus: 'trace' + - name: 'skill-fired' + type: 'tool_used' + tool: 'Skill' + input_match: '"skill"\s*:\s*"(?:harness-ops:)?plugins-audit"' +context: + scaffold_script: scaffold.sh diff --git a/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/prompt.md b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/prompt.md new file mode 100644 index 00000000000..93df75c248d --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/prompt.md @@ -0,0 +1,8 @@ +--- +description: "The scaffold seeds a fixture fleet in the run's home (a marketplace, its catalog, install records and settings) so the default marketplace resolves. Needs --scaffold --allow-tools Bash, with the real node first on PATH: a mise shim on PATH fails under the run's temporary home." +tags: ['plugins-audit'] +allowed_tools: [Read, Glob, Grep, Skill, Bash] +expected_outcome: 'The skill runs sync-run.sh with --audit (never --journal-root, --pull-source or --only-install), pastes the rendered prediction, and tells the user that applying it is /harness-ops:plugins sync, which they run themselves. It reports a zero-update prediction as a lower bound against the unrefreshed catalog, not as proof the fleet is current.' +--- + +Are my plugins current? Don't change anything yet. diff --git a/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/expectation-1.json b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/expectation-1.json new file mode 100644 index 00000000000..ea5a8afe874 --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/expectation-1.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The skill runs sync-run.sh with --audit (never --journal-root, --pull-source or --only-install), pastes the rendered prediction, and tells the user that applying it is /harness-ops:plugins sync, which they run themselves. It reports a zero-update prediction as a lower bound against the unrefreshed catalog, not as proof the fleet is current.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/expectation-2.json b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/expectation-2.json new file mode 100644 index 00000000000..ea5a8afe874 --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/expectation-2.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The skill runs sync-run.sh with --audit (never --journal-root, --pull-source or --only-install), pastes the rendered prediction, and tells the user that applying it is /harness-ops:plugins sync, which they run themselves. It reports a zero-update prediction as a lower bound against the unrefreshed catalog, not as proof the fleet is current.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/expectation-3.json b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/expectation-3.json new file mode 100644 index 00000000000..ea5a8afe874 --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/expectation-3.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The skill runs sync-run.sh with --audit (never --journal-root, --pull-source or --only-install), pastes the rendered prediction, and tells the user that applying it is /harness-ops:plugins sync, which they run themselves. It reports a zero-update prediction as a lower bound against the unrefreshed catalog, not as proof the fleet is current.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/skill-fired.json b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/skill-fired.json new file mode 100644 index 00000000000..aa932f6e6d9 --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/samples/skill-fired.json @@ -0,0 +1,54 @@ +{ + "pass": [ + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "harness-ops:plugins-audit" + } + } + ], + "why": "qualified name" + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "plugins-audit" + } + } + ], + "why": "bare name" + } + ], + "fail": [ + { + "answer": [], + "why": "no Skill call" + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "other-plugin:other-skill" + } + } + ], + "why": "another skill" + }, + { + "answer": [ + { + "tool": "Skill", + "input": { + "skill": "harness-ops:plugins-audit-other" + } + } + ], + "why": "a skill whose name only starts the same" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/scaffold.sh b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/scaffold.sh new file mode 100755 index 00000000000..b47b1b32ecb --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-model-starts-audit-not-sync/scaffold.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Seeds a fixture fleet in the run's home: one marketplace, one plugin behind its catalog, one current, one new. +# The runner sets HOME to the run's temporary home for a scaffold script +# (https://code.claude.com/docs/en/plugin-evals, "Set up fixtures and mocks"). +# The guard refuses the account's real home and never overwrites a file. +set -euo pipefail + +refuse() { + echo "refusing: $1" >&2 + exit 1 +} + +[[ -n "${HOME:-}" && -d "$HOME" ]] || refuse "HOME is unset or not a directory" +user=$(id -un) +if command -v getent >/dev/null 2>&1; then + real_home=$(getent passwd "$user" | cut -d: -f6) +elif command -v dscl >/dev/null 2>&1; then + real_home=$(dscl . -read "/Users/$user" NFSHomeDirectory | awk '{print $2}') +else + refuse "cannot look up the account's real home to compare against" +fi +[[ -n "$real_home" ]] || refuse "the account's real home did not resolve" +[[ "$(cd "$HOME" && pwd -P)" != "$(cd "$real_home" && pwd -P)" ]] || refuse "HOME is the account's real home: $HOME" + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +p="$HOME/.claude/plugins" +m="$p/marketplaces/fixture-market" +targets=("$p/known_marketplaces.json" "$p/installed_plugins.json" "$HOME/.claude/settings.json" + "$HOME/.claude/settings.local.json" "$PWD/.claude/settings.local.json" "$PWD/.claude/settings.json") +for f in "${targets[@]}" "$m"; do [[ ! -e "$f" ]] || refuse "$f already exists"; done + +mkdir -p "$m/.claude-plugin" "$p/cache" "$PWD/.claude" +for spec in alpha:2.0.0 beta:1.0.0 gamma:1.0.0 harness-ops:0.0.0; do + n=${spec%%:*} + v=${spec##*:} + mkdir -p "$m/$n/.claude-plugin" + printf '{"name":"%s","version":"%s"}\n' "$n" "$v" >"$m/$n/.claude-plugin/plugin.json" +done +cat >"$m/.claude-plugin/marketplace.json" <<'JSON' +{"name":"fixture-market","owner":{"name":"fixture"},"plugins":[ +{"name":"alpha","source":"./alpha"}, +{"name":"beta","source":"./beta"}, +{"name":"gamma","source":"./gamma"}, +{"name":"harness-ops","source":"./harness-ops"}]} +JSON +cat >"$p/known_marketplaces.json" <"$p/installed_plugins.json" <"$HOME/.claude/settings.json" +for f in "$HOME/.claude/settings.local.json" "$PWD/.claude/settings.local.json" "$PWD/.claude/settings.json"; do printf '{}\n' >"$f"; done diff --git a/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/case.yaml b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/case.yaml new file mode 100644 index 00000000000..4195ec6841e --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/case.yaml @@ -0,0 +1,23 @@ +schema_version: "1.1" +name: 'plugins-audit-rendered-report-is-pasted-not-restated' +graders: + - name: 'expectation-1' + type: 'llm' + criteria: 'The response reproduces the rendered report verbatim rather than summarizing it in its own words' + focus: 'trace' + - name: 'expectation-2' + type: 'llm' + criteria: 'The response does not restate a number, id, scope, or version that the render already printed (no separate ''three plugins would be updated'' sentence, no re-listing of gamma@market1''s versions)' + focus: 'trace' + - name: 'expectation-3' + type: 'llm' + criteria: 'The response does not add a row, section, or count the render omitted, and does not recompute the Divergences split from the digest' + focus: 'trace' + - name: 'expectation-4' + type: 'llm' + criteria: 'The only line the response adds after the render is the reload guidance, stated as the docs'' two-step: bare /reload-plugins first, /reload-plugins --force only if the bare command warns about re-reading the conversation' + focus: 'trace' + - name: 'expectation-5' + type: 'llm' + criteria: 'The response does not append its own monitor restart call-out; that bullet comes from the render''s Action needed when updated_with_monitors is non-empty' + focus: 'trace' diff --git a/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/prompt.md b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/prompt.md new file mode 100644 index 00000000000..fe5e1f76631 --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/prompt.md @@ -0,0 +1,7 @@ +--- +tags: ['plugins-audit'] +allowed_tools: [Read, Glob, Grep, Skill, Bash] +expected_outcome: 'The response pastes the rendered report as printed and appends exactly one line of its own: the reload guidance, stated as the docs'' two-step (run /reload-plugins; if it warns that the next message would re-read the conversation, run /reload-plugins --force). It does not retype, paraphrase, or recompute any count, id, scope, or version the render already carries (the 3, the 1, gamma''s versions, the 2 and 7 of the Divergences line), does not add rows the render omitted, and does not compute its own Divergences split from the digest. It also does not add a monitor call-out of its own: that bullet is the render''s, under Action needed, when the digest''s updated_with_monitors is non-empty.' +--- + +/harness-ops:plugins-audit — sync-run.sh --audit --render printed the digest line and then the rendered report, whose rows read `Would update: 3 plugin(s)` with three `would run: claude plugin update … -s user` lines, `Would withhold: 1 downgrade(s)` naming gamma@market1 0.5.0 → 0.1.0, `Divergences: 2 behind here → run /harness-ops:plugins converge; on this whole machine 7 actionable: …`, and a `Timing:` row. diff --git a/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-1.json b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-1.json new file mode 100644 index 00000000000..815f1dbf6ae --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-1.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The response pastes the rendered report as printed and appends exactly one line of its own: the reload guidance, stated as the docs' two-step (run /reload-plugins; if it warns that the next message would re-read the conversation, run /reload-plugins --force). It does not retype, paraphrase, or recompute any count, id, scope, or version the render already carries (the 3, the 1, gamma's versions, the 2 and 7 of the Divergences line), does not add rows the render omitted, and does not compute its own Divergences split from the digest. It also does not add a monitor call-out of its own: that bullet is the render's, under Action needed, when the digest's updated_with_monitors is non-empty.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-2.json b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-2.json new file mode 100644 index 00000000000..815f1dbf6ae --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-2.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The response pastes the rendered report as printed and appends exactly one line of its own: the reload guidance, stated as the docs' two-step (run /reload-plugins; if it warns that the next message would re-read the conversation, run /reload-plugins --force). It does not retype, paraphrase, or recompute any count, id, scope, or version the render already carries (the 3, the 1, gamma's versions, the 2 and 7 of the Divergences line), does not add rows the render omitted, and does not compute its own Divergences split from the digest. It also does not add a monitor call-out of its own: that bullet is the render's, under Action needed, when the digest's updated_with_monitors is non-empty.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-3.json b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-3.json new file mode 100644 index 00000000000..815f1dbf6ae --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-3.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The response pastes the rendered report as printed and appends exactly one line of its own: the reload guidance, stated as the docs' two-step (run /reload-plugins; if it warns that the next message would re-read the conversation, run /reload-plugins --force). It does not retype, paraphrase, or recompute any count, id, scope, or version the render already carries (the 3, the 1, gamma's versions, the 2 and 7 of the Divergences line), does not add rows the render omitted, and does not compute its own Divergences split from the digest. It also does not add a monitor call-out of its own: that bullet is the render's, under Action needed, when the digest's updated_with_monitors is non-empty.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-4.json b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-4.json new file mode 100644 index 00000000000..815f1dbf6ae --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-4.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The response pastes the rendered report as printed and appends exactly one line of its own: the reload guidance, stated as the docs' two-step (run /reload-plugins; if it warns that the next message would re-read the conversation, run /reload-plugins --force). It does not retype, paraphrase, or recompute any count, id, scope, or version the render already carries (the 3, the 1, gamma's versions, the 2 and 7 of the Divergences line), does not add rows the render omitted, and does not compute its own Divergences split from the digest. It also does not add a monitor call-out of its own: that bullet is the render's, under Action needed, when the digest's updated_with_monitors is non-empty.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-5.json b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-5.json new file mode 100644 index 00000000000..815f1dbf6ae --- /dev/null +++ b/plugins/harness-ops/evals/plugins-audit-rendered-report-is-pasted-not-restated/samples/expectation-5.json @@ -0,0 +1,8 @@ +{ + "pass": [ + { + "answer": "The response pastes the rendered report as printed and appends exactly one line of its own: the reload guidance, stated as the docs' two-step (run /reload-plugins; if it warns that the next message would re-read the conversation, run /reload-plugins --force). It does not retype, paraphrase, or recompute any count, id, scope, or version the render already carries (the 3, the 1, gamma's versions, the 2 and 7 of the Divergences line), does not add rows the render omitted, and does not compute its own Divergences split from the digest. It also does not add a monitor call-out of its own: that bullet is the render's, under Action needed, when the digest's updated_with_monitors is non-empty.", + "why": "the source's expected_output, which describes the reply rather than recording one; confirm the label before calibrating" + } + ] +} diff --git a/plugins/harness-ops/skills/plugins-audit/evals/evals.json b/plugins/harness-ops/skills/plugins-audit/evals/evals.json deleted file mode 100644 index b12768f49a1..00000000000 --- a/plugins/harness-ops/skills/plugins-audit/evals/evals.json +++ /dev/null @@ -1,58 +0,0 @@ -{ - "skill_name": "plugins-audit", - "evals": [ - { - "id": 1, - "name": "audit-issues-zero-mutating-calls", - "prompt": "/harness-ops:plugins-audit", - "expected_output": "audit runs the same algorithm as sync but every mutating CLI call becomes a prediction: no `claude plugin install|update|uninstall|marketplace update` command is ever run, and normalize-enabled-plugins.sh is invoked with --check rather than written. The reads those calls sit beside still run. Step 1's pre-refresh fleet-state.sh snapshot is taken (into the run's throwaway scratch directory) so the catalog-regression check keeps its first interval, and only the `claude plugin marketplace update` beside it is replaced by a prediction. The response calls fleet-state.sh (read-only) and reports what sync/converge would do without doing it.", - "files": [], - "expectations": [ - "The response issues zero `claude plugin install|update|uninstall` or `claude plugin marketplace update` invocations", - "The response still computes state from the read-only fleet-state.sh reads, which sync-run.sh --audit takes", - "The response still takes Step 1's pre-refresh fleet-state.sh snapshot before declining the refresh: it does not treat the whole of Step 1 as skipped, and the catalog-regression check still has its pre-refresh to pre interval available", - "The response frames its output as a prediction of what `sync` (and, where relevant, `converge`) would do, not as work already performed", - "If the response mentions enabledPlugins key order, it uses normalize-enabled-plugins.sh --check (or equivalent prediction) and never writes ~/.claude/settings.json or a committed project settings file" - ] - }, - { - "id": 2, - "name": "rendered-report-is-pasted-not-restated", - "prompt": "/harness-ops:plugins-audit — sync-run.sh --audit --render printed the digest line and then the rendered report, whose rows read `Would update: 3 plugin(s)` with three `would run: claude plugin update … -s user` lines, `Would withhold: 1 downgrade(s)` naming gamma@market1 0.5.0 → 0.1.0, `Divergences: 2 behind here → run /harness-ops:plugins converge; on this whole machine 7 actionable: …`, and a `Timing:` row.", - "expected_output": "The response pastes the rendered report as printed and appends exactly one line of its own: the reload guidance, stated as the docs' two-step (run /reload-plugins; if it warns that the next message would re-read the conversation, run /reload-plugins --force). It does not retype, paraphrase, or recompute any count, id, scope, or version the render already carries (the 3, the 1, gamma's versions, the 2 and 7 of the Divergences line), does not add rows the render omitted, and does not compute its own Divergences split from the digest. It also does not add a monitor call-out of its own: that bullet is the render's, under Action needed, when the digest's updated_with_monitors is non-empty.", - "files": [], - "expectations": [ - "The response reproduces the rendered report verbatim rather than summarizing it in its own words", - "The response does not restate a number, id, scope, or version that the render already printed (no separate 'three plugins would be updated' sentence, no re-listing of gamma@market1's versions)", - "The response does not add a row, section, or count the render omitted, and does not recompute the Divergences split from the digest", - "The only line the response adds after the render is the reload guidance, stated as the docs' two-step: bare /reload-plugins first, /reload-plugins --force only if the bare command warns about re-reading the conversation", - "The response does not append its own monitor restart call-out; that bullet comes from the render's Action needed when updated_with_monitors is non-empty" - ] - }, - { - "id": 3, - "name": "model-starts-audit-not-sync", - "prompt": "Are my plugins current? Don't change anything yet.", - "expected_output": "The skill runs sync-run.sh with --audit (never --journal-root, --pull-source or --only-install), pastes the rendered prediction, and tells the user that applying it is /harness-ops:plugins sync, which they run themselves. It reports a zero-update prediction as a lower bound against the unrefreshed catalog, not as proof the fleet is current.", - "files": [], - "expectations": [ - "The script call carries --audit and none of --journal-root, --pull-source or --only-install", - "The response does not run or start /harness-ops:plugins sync or converge itself; it names sync as the user's next step", - "A prediction of zero updates is framed as a lower bound against the unrefreshed catalog, not as the fleet being current" - ] - }, - { - "id": 4, - "name": "divergence-report-collapses-bulk-names-conflicts", - "prompt": "/harness-ops:plugins-audit — I have 29 plugins with an actionable version divergence between project and user scope, plus 2 more that are multi-scope but pinned to the identical version, and one plugin id that fleet-state.sh reports under delisted_settings_only (true in enabledPlugins, no install record, and no matching catalog entry).", - "expected_output": "The report's Divergences line collapses the 29 actionable (version-behind) entries into one bulk count and names `/harness-ops:plugins converge` as the fix — it does NOT list all 29 individually (at most one sub-line per project path, each naming that path's record count and up to 3 ids, then `+K more` when more ids remain), and it does NOT count the 2 same-version multi-scope installs as actionable (versionsMatch: true is benign). The one delisted_settings_only plugin id gets its own per-row detail line under Action needed as a genuine conflict, distinct from the bulk divergence count.", - "files": [], - "expectations": [ - "The 29 actionable divergences are collapsed into a single bulk-count line, not enumerated one by one; per-project-path sub-lines under it are allowed, each naming at most 3 ids, followed by `+K more` when more ids remain", - "The 2 same-version (versionsMatch: true) multi-scope installs are excluded from the actionable divergence count", - "The delisted_settings_only plugin id gets an individual per-row detail line under Action needed, separate from the bulk divergence line", - "The response never claims to detect or report an enable-state mismatch (true in one scope, false in another) — fleet-state.sh only exposes the merged effective value, not each scope's raw enabledPlugins map" - ] - } - ] -}