From 617d750f2aeb0f2b8be82b43041fb4a0b3d33f46 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Wed, 15 Jul 2026 03:28:59 -0400 Subject: [PATCH 1/7] feat: restore tier-3 eval coverage across eight plugins Recreates the eval/test-coverage attrition from the salvage sweep (items #13, #15-#23 plus the flagged review-toolkit and docs-hygiene items), genericized and validated against evals.schema.json: - songwriting 0.4.0: 13 behavioral evals mapped onto the multi-skill split - ai-briefing 0.4.0: 6 engine evals + 3 synthetic fixtures via the audience-defaults seam - event-storming 0.4.0: offline board-export eval + fixture for --discover-bcs (disjoint from the live-Miro eval) - codebase-audit 0.3.0: scope-boundary routing eval - discovery 0.5.0: research floor-scaling + broad-topic-minimums evals - source-control 0.2.0: readiness security-gate (+fixture), mixed-actor, and three worktree evals - docs-hygiene 0.4.0: self-contained compress/declutter fixtures (empirically verified against detect.sh), rename-references eval-case clauses - review-toolkit 0.6.0: fanout evals 6 -> 20 (dedup/severity, fix-pass safety fence, run-everything reconciliation/ordering) Per-plugin version bumps + CHANGELOG entries; all validations pass. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011xHkkNc7CR98L8Xz9Mu7ZA --- .../ai-briefing/.claude-plugin/plugin.json | 2 +- plugins/ai-briefing/CHANGELOG.md | 20 ++ .../skills/ai-briefing/evals/evals.json | 86 +++++++++ .../evals/fixtures/archive-sample.md | 44 +++++ .../evals/fixtures/candidate-items-sample.md | 16 ++ .../evals/fixtures/open-window-sample.md | 31 ++++ .../codebase-audit/.claude-plugin/plugin.json | 2 +- plugins/codebase-audit/CHANGELOG.md | 10 + .../skills/codebase-audit/evals/evals.json | 13 ++ plugins/discovery/.claude-plugin/plugin.json | 2 +- plugins/discovery/CHANGELOG.md | 11 ++ .../skills/research/evals/evals.json | 41 +++++ .../docs-hygiene/.claude-plugin/plugin.json | 2 +- plugins/docs-hygiene/CHANGELOG.md | 12 ++ .../skills/compress/evals/evals.json | 19 +- .../evals/fixtures/audit-fixture-dir/lean.md | 19 ++ .../evals/fixtures/audit-fixture-dir/mixed.md | 20 ++ .../fixtures/audit-fixture-dir/verbose.md | 25 +++ .../skills/declutter/evals/evals.json | 15 ++ .../declutter/evals/fixtures/legit-optouts.md | 49 +++++ .../skills/rename-references/SKILL.md | 4 +- .../event-storming/.claude-plugin/plugin.json | 2 +- plugins/event-storming/CHANGELOG.md | 6 + .../skills/simulation/evals/evals.json | 17 ++ .../fixtures/big-picture-board-export.md | 74 ++++++++ .../review-toolkit/.claude-plugin/plugin.json | 2 +- plugins/review-toolkit/CHANGELOG.md | 14 ++ .../code-review-fanout/evals/evals.json | 171 ++++++++++++++++++ .../songwriting/.claude-plugin/plugin.json | 2 +- plugins/songwriting/CHANGELOG.md | 14 ++ .../skills/co-write/evals/evals.json | 34 ++++ .../skills/diagnosis/evals/evals.json | 47 +++++ .../skills/object-writing/evals/evals.json | 19 ++ .../songwriting/skills/rhyme/evals/evals.json | 33 ++++ .../skills/song-form/evals/evals.json | 32 ++++ .../skills/workflow/evals/evals.json | 45 +++++ .../source-control/.claude-plugin/plugin.json | 2 +- plugins/source-control/CHANGELOG.md | 10 + .../skills/pull-request/evals/evals.json | 26 +++ .../evals/fixtures/ci-status-security-fail.md | 44 +++++ .../skills/worktree/evals/evals.json | 39 ++++ 41 files changed, 1061 insertions(+), 15 deletions(-) create mode 100644 plugins/ai-briefing/CHANGELOG.md create mode 100644 plugins/ai-briefing/skills/ai-briefing/evals/evals.json create mode 100644 plugins/ai-briefing/skills/ai-briefing/evals/fixtures/archive-sample.md create mode 100644 plugins/ai-briefing/skills/ai-briefing/evals/fixtures/candidate-items-sample.md create mode 100644 plugins/ai-briefing/skills/ai-briefing/evals/fixtures/open-window-sample.md create mode 100644 plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/lean.md create mode 100644 plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/mixed.md create mode 100644 plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/verbose.md create mode 100644 plugins/docs-hygiene/skills/declutter/evals/fixtures/legit-optouts.md create mode 100644 plugins/event-storming/skills/simulation/evals/fixtures/big-picture-board-export.md create mode 100644 plugins/songwriting/skills/co-write/evals/evals.json create mode 100644 plugins/songwriting/skills/diagnosis/evals/evals.json create mode 100644 plugins/songwriting/skills/object-writing/evals/evals.json create mode 100644 plugins/songwriting/skills/rhyme/evals/evals.json create mode 100644 plugins/songwriting/skills/song-form/evals/evals.json create mode 100644 plugins/songwriting/skills/workflow/evals/evals.json create mode 100644 plugins/source-control/CHANGELOG.md create mode 100644 plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md diff --git a/plugins/ai-briefing/.claude-plugin/plugin.json b/plugins/ai-briefing/.claude-plugin/plugin.json index d4930177f..b4a8f814f 100644 --- a/plugins/ai-briefing/.claude-plugin/plugin.json +++ b/plugins/ai-briefing/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "ai-briefing", - "version": "0.3.1", + "version": "0.4.0", "description": "Build source-backed AI-industry briefings from official vendor publications, configured RSS/Atom feeds, GitHub releases, reputable secondary reporting, and user-supplied URLs. Deduplicate, rank, and present results as markdown or optional HTML/PPTX decks, with repository-owned profile, audience, and brand configuration. Automated X/Twitter collection is disabled; Playwright is used only for deterministic local rendering.", "author": { "name": "Melodic Software", diff --git a/plugins/ai-briefing/CHANGELOG.md b/plugins/ai-briefing/CHANGELOG.md new file mode 100644 index 000000000..113e6c3fa --- /dev/null +++ b/plugins/ai-briefing/CHANGELOG.md @@ -0,0 +1,20 @@ +# Changelog + +All notable changes to the `ai-briefing` plugin are documented here. Format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. + +## [0.4.0] + +### Added + +- Engine behavioral evals in `skills/ai-briefing/evals/evals.json`, covering: `retro` action + routing and per-item acted/noted/skipped scoring against an archived briefing; `search` + action full-text matching across archives; markdown-only output when `--format slides`/`html` + is not explicitly requested; merge-not-append behavior when folding newly collected items into + an already-open briefing window; the apolitical filter, pragmatic-use ranking lens, and + profile-provided impact-lens annotation (via `references/audience-defaults.md`); and graceful, + visibly-surfaced degradation when an optional collection source is unreachable, without + aborting the run. +- Three supporting fixtures under `skills/ai-briefing/evals/fixtures/`: `archive-sample.md`, + `open-window-sample.md`, and `candidate-items-sample.md` — neutral, synthetic AI-industry + content with no real company, person, or consumer-specific references. diff --git a/plugins/ai-briefing/skills/ai-briefing/evals/evals.json b/plugins/ai-briefing/skills/ai-briefing/evals/evals.json new file mode 100644 index 000000000..a3f021afd --- /dev/null +++ b/plugins/ai-briefing/skills/ai-briefing/evals/evals.json @@ -0,0 +1,86 @@ +{ + "skill_name": "ai-briefing", + "evals": [ + { + "id": 1, + "name": "retro-action-routing-and-scoring", + "prompt": "/ai-briefing retro --meeting 5\n\nThe archived briefing for meeting 5 is at evals/fixtures/archive-sample.md relative to the skill directory — read it, then run the retro.", + "expected_output": "An interactive retrospective that reads the meeting-5 archive, walks each item, captures an acted/noted/skipped signal per item, and produces a retro annotation for meeting 5 (e.g., a retro-5 record under the profile's state directory) — without launching a fresh collection run or reopening the window.", + "files": ["evals/fixtures/archive-sample.md"], + "expectations": [ + "Routes to the retro action — scores already-archived items and does NOT start a fresh collection run (no vendor-blog/GitHub/RSS/secondary-reporting wave collection)", + "Extracts the individual item bullets (provider, headline, source URL) from the meeting-5 archive fixture rather than inventing items", + "Captures an acted / noted / skipped signal for each item (interactive per-item prompting, or a documented auto-mode heuristic inference)", + "Produces retro output (a per-item annotation plus a pattern summary), not a new briefing for a different window", + "Surfaces follow-list demote/promote recommendations derived from the per-item signals, rather than emitting a fresh collection briefing" + ] + }, + { + "id": 2, + "name": "search-action-across-archives", + "prompt": "/ai-briefing search \"Cloud Agents\"\n\nTreat evals/fixtures/archive-sample.md (relative to the skill directory) as the archived-briefing corpus to search.", + "expected_output": "A full-text search result that returns the matching Northwind Cloud Agents item from the archive, grouped by meeting number with tier/signal context, without running any collection.", + "files": ["evals/fixtures/archive-sample.md"], + "expectations": [ + "Routes to the search action — full-text search over archived briefings, no collection run and no new briefing emitted", + "Returns the matching item(s) that actually contain the query text in the fixture (the Northwind Cloud Agents GA item), grouped or labelled by meeting number", + "Reports match context such as tier (HIGH/MED/LOW) and/or cross-meeting recurrence count", + "Does not fabricate items absent from the archive — only real matches from the fixture are returned" + ] + }, + { + "id": 3, + "name": "slides-never-auto-generated", + "prompt": "Catch me up on AI from the last two weeks so I'm ready for the team meeting.", + "expected_output": "A markdown briefing covering roughly the last 14 days. No PPTX or HTML deck is produced because the user did not opt into --format slides or --format html.", + "files": [], + "expectations": [ + "Output is a markdown briefing, not a PPTX or HTML slide deck", + "The slide/HTML build pipeline (output/build/*.js) is not invoked without an explicit --format slides or --format html opt-in", + "The window resolves to roughly 14 days (interpreting 'last two weeks' / the --since default) rather than a 1-day or 30-day window" + ] + }, + { + "id": 4, + "name": "merge-not-append-continuation", + "prompt": "The meeting-prep briefing already open for this window is at evals/fixtures/open-window-sample.md relative to the skill directory — the bottom section lists new items collected today. Fold the new items into the briefing and give me the updated consolidated document.", + "expected_output": "A single consolidated briefing where the new items are merged into the existing provider buckets under their HIGH/MED/LOW headings, prior items preserved, and no delta/supplement section appended.", + "files": ["evals/fixtures/open-window-sample.md"], + "expectations": [ + "New items are merged into the existing provider buckets under their HIGH/MED/LOW headings (add-only = MERGE)", + "No '## Run N', '## Supplement', or 'New items collected today' delta/append section survives in the merged output — the body stays a single consolidated document", + "Existing items from the prior briefing are preserved in the merged output, not dropped", + "The previously-empty Northwind Systems bucket is populated by its new item rather than left as a stale 'no notable items' note", + "Each provider appears once with its items consolidated under it, not duplicated across multiple sections" + ] + }, + { + "id": 5, + "name": "apolitical-filter-pragmatic-ranking-bucket-coverage", + "prompt": "I've already collected this raw batch of AI items — evals/fixtures/candidate-items-sample.md relative to the skill directory. The active profile's audience.md declares an impact lens over a typical web-application stack (a JS/TS runtime, a hosted database, an auth provider, and cloud hosting). Categorize, apply the filters, rank them, and lay out as the briefing.", + "expected_output": "A ranked briefing that drops partisan-only items, keeps genuine industry controversy even when a lawmaker is named, ranks usable tooling releases HIGH above research speculation, routes the robotics item to EXTRAS, and tags HIGH items with an impact annotation against the declared stack lens.", + "files": ["evals/fixtures/candidate-items-sample.md"], + "expectations": [ + "Partisan-only items are dropped before ranking (the candidate-deepfake meme and the partisan horse-race thread do not appear anywhere in the briefing)", + "Genuine industry-wide controversy is kept even though a lawmaker is named (the VendorCo procurement-dispute item appears, under Legal & regulatory)", + "Tooling/IDE/CLI releases usable within a week are ranked HIGH (the Solstice Code, Meridian Agent CLI, and Northwind IDE items) ahead of research speculation", + "The research-speculation post and the hype post are ranked LOW/MED, not HIGH", + "The robotics item (the Anchorpoint Unit-7 warehouse demo) is routed to the EXTRAS section rather than a mainstream provider bucket", + "HIGH items carry an impact annotation assessed against the declared stack lens (e.g., \"impact: medium — ...\"), per the profile-provided impact lens" + ] + }, + { + "id": 6, + "name": "unreachable-source-degrades-visibly", + "prompt": "/ai-briefing --since 3d\n\n(One of the profile's configured RSS/Atom feeds times out and is unreachable during this run; official vendor blogs and GitHub releases remain reachable.)", + "expected_output": "The briefing proceeds using the still-reachable source classes; the unreachable feed is surfaced as a visible warning/caveat rather than silently dropped, and the run is not aborted for one unreachable optional source.", + "files": [], + "expectations": [ + "The run continues via the still-reachable source classes — one unreachable RSS/Atom feed does NOT block or abort the briefing", + "The unreachable source is surfaced as a visible warning (in the run report and/or the briefing's caveats section), not silently ignored", + "The run is not failed outright for a single unreachable optional source, matching the documented 'no black-box degradation' quality gate", + "Items collected from the still-reachable source classes are still deduplicated, ranked, and emitted normally in the resulting briefing" + ] + } + ] +} diff --git a/plugins/ai-briefing/skills/ai-briefing/evals/fixtures/archive-sample.md b/plugins/ai-briefing/skills/ai-briefing/evals/fixtures/archive-sample.md new file mode 100644 index 000000000..8ceca3f40 --- /dev/null +++ b/plugins/ai-briefing/skills/ai-briefing/evals/fixtures/archive-sample.md @@ -0,0 +1,44 @@ +# AI Briefing — Meeting 5 (archived) + +Window: 2026-06-05 to 2026-06-19 · Runs: 3 · Source classes: official vendor blogs, GitHub releases, RSS/Atom feeds, secondary reporting + +## Solstice AI + +### HIGH + +- **Solstice Code v3.4 ships checkpoint rewind** — restore any prior turn's file state without external version control. Source: Solstice AI Engineering Blog, (2026-06-06). +- **Solstice model pricing cut 20%** — input/output token cost drops across all paid tiers. impact: medium — lowers per-run inference cost. Source: Solstice AI Pricing Notes, (2026-06-09). + +### MED + +- Solstice AI publishes a research note on interpretability via feature steering. Source: Solstice AI Research Blog, (2026-06-11). + +## Meridian Labs + +### HIGH + +- **Meridian Agent CLI adds MCP server support** — call local MCP tools from the terminal agent. Source: Meridian Labs Release Notes (GitHub), (2026-06-12). + +### LOW + +- A Meridian Labs executive teases "a big announcement soon" with no specifics. Source: secondary reporting, (2026-06-13). + +## Northwind Systems + +### HIGH + +- **Northwind Cloud Agents GA** — background agents run on Northwind-hosted infrastructure and open pull requests directly. Source: Northwind Systems Blog, (2026-06-15). + +## Legal & regulatory + +### HIGH + +- **Cross-border AI-governance framework finalized** — general-purpose model obligations take effect next quarter, industry-wide compliance impact. Source: secondary reporting corroborated by the regulator's official notice, (2026-06-17). + +## Other + +*No notable items this window.* + +## EXTRAS (Robotics, science, and novel applications) + +- Anchorpoint Dynamics' Unit-7 humanoid demo folds laundry unassisted in an unstructured home setting. Source: Anchorpoint Dynamics Blog, (2026-06-18). diff --git a/plugins/ai-briefing/skills/ai-briefing/evals/fixtures/candidate-items-sample.md b/plugins/ai-briefing/skills/ai-briefing/evals/fixtures/candidate-items-sample.md new file mode 100644 index 000000000..07dc8e8a0 --- /dev/null +++ b/plugins/ai-briefing/skills/ai-briefing/evals/fixtures/candidate-items-sample.md @@ -0,0 +1,16 @@ +# Raw candidate items — pre-categorize batch + +Unranked, unfiltered items collected this run via approved source classes (official vendor +blogs, GitHub releases, RSS/Atom feeds, secondary reporting). Categorize, apply the apolitical +filter, rank HIGH/MED/LOW by the pragmatic-use lens, and lay out as the briefing. + +1. Solstice Code v3.6 released: native subagent checkpointing, install via update. Engineers can use it this week. Source: Solstice AI Engineering Blog, (2026-06-23). +2. Meridian Agent CLI 0.9 adds `--effort` flag and desktop app parity. Source: Meridian Labs Release Notes (GitHub), (2026-06-23). +3. Northwind IDE 4.1 ships inline Cloud Agent handoff from the editor. Source: Northwind Systems Blog, (2026-06-23). +4. A Meridian Labs research blog post speculates scaling laws may bend at very large parameter counts; no release, purely theoretical. Source: Meridian Labs Research Blog, (2026-06-22). +5. VendorCo and a national defense procurement office dispute a supply-chain security clause in a multi-year AI procurement contract; a lawmaker is quoted in the coverage. Affects federal AI access industry-wide. Source: secondary reporting, (2026-06-21). +6. A deepfake video of a political candidate saying absurd things circulates online; pure partisan meme, high engagement. Source: secondary reporting, (2026-06-20). +7. A widely shared thread argues one political party uses AI campaign tools better than the other; partisan horse-race framing, no industry substance. Source: secondary reporting, (2026-06-20). +8. An online influencer posts "AGI is basically here, feel the acceleration" with no supporting artifact. Source: secondary reporting, (2026-06-19). +9. Anchorpoint Dynamics' Unit-7 humanoid completes a full warehouse pick-and-pack shift autonomously. Source: Anchorpoint Dynamics Blog, (2026-06-19). +10. Cobalt Dynamics posts open weights for its 4.1 open model with a minor context-length bump. Source: Cobalt Dynamics Blog, (2026-06-18). diff --git a/plugins/ai-briefing/skills/ai-briefing/evals/fixtures/open-window-sample.md b/plugins/ai-briefing/skills/ai-briefing/evals/fixtures/open-window-sample.md new file mode 100644 index 000000000..017561602 --- /dev/null +++ b/plugins/ai-briefing/skills/ai-briefing/evals/fixtures/open-window-sample.md @@ -0,0 +1,31 @@ +# AI Briefing — Meeting 6 (open window) + +Window: 2026-06-19 to present · Runs: 1 · Status: open (not yet closed out) + +## Solstice AI + +### HIGH + +- **Solstice Code v3.5 ships background tasks** — long-running agents survive session end. Source: Solstice AI Engineering Blog, (2026-06-20). + +### MED + +- Solstice AI's MCP registry adds 40 verified servers. Source: Solstice AI Blog, (2026-06-21). + +## Meridian Labs + +### HIGH + +- **Meridian Desktop app enters public beta.** Source: Meridian Labs Blog, (2026-06-22). + +## Northwind Systems + +*No notable items this window.* + +--- + +## New items collected today (uncategorized — fold these in) + +- Solstice Code v3.6 adds native subagent checkpointing (HIGH, tooling). Source: Solstice AI Engineering Blog, (2026-06-23). +- Northwind IDE 4.1 ships inline Cloud Agent handoff (HIGH, tooling). Source: Northwind Systems Blog, (2026-06-23). +- Meridian Agent CLI 0.9 adds an `--effort` flag (MED, tooling). Source: Meridian Labs Release Notes (GitHub), (2026-06-23). diff --git a/plugins/codebase-audit/.claude-plugin/plugin.json b/plugins/codebase-audit/.claude-plugin/plugin.json index 9f79efb6f..1bb22a083 100644 --- a/plugins/codebase-audit/.claude-plugin/plugin.json +++ b/plugins/codebase-audit/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "codebase-audit", - "version": "0.2.0", + "version": "0.3.0", "description": "Repo-wide drift audit between docs, config, code, and architecture: verifies every factual claim against reality via parallel subagent fan-out, severity-rates findings, and fixes or presents for review. Audit dimensions are configurable through a tracked .claude/codebase-audit.md config file written by the setup skill.", "author": { "name": "Melodic Software", diff --git a/plugins/codebase-audit/CHANGELOG.md b/plugins/codebase-audit/CHANGELOG.md index 2edd0c4c5..5bc0daf3f 100644 --- a/plugins/codebase-audit/CHANGELOG.md +++ b/plugins/codebase-audit/CHANGELOG.md @@ -3,6 +3,16 @@ All notable changes to the `codebase-audit` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.3.0] + +### Added + +- Eval covering the scope-boundary decline: declining claim-extraction fan-out over + `settings.json` / `.mcp.json` / hooks / permissions and routing to the adjacent + `claude-config-audit` plugin's `/claude-config-audit:settings-audit` skill (or stating + out-of-scope when that plugin is not installed) — behavior already documented in SKILL.md, + now regression-tested. + ## [0.2.0] ### Added diff --git a/plugins/codebase-audit/skills/codebase-audit/evals/evals.json b/plugins/codebase-audit/skills/codebase-audit/evals/evals.json index 903672132..97aa8cc86 100644 --- a/plugins/codebase-audit/skills/codebase-audit/evals/evals.json +++ b/plugins/codebase-audit/skills/codebase-audit/evals/evals.json @@ -62,6 +62,19 @@ "Presents a verified non-issues list showing what was checked", "When no discrepancies are found, states explicitly that all claims verified as correct rather than padding with marginal issues" ] + }, + { + "id": 6, + "name": "boundary-route-to-settings-audit", + "prompt": "/codebase-audit:codebase-audit\n\nCheck my .claude/settings.json and .mcp.json for schema drift, and tell me whether my hooks and permissions are misconfigured.", + "expected_output": "Recognizes that settings.json / .mcp.json schema-drift and hook/permission validation is the adjacent claude-config-audit lane, not this skill's factual-claim-drift lane. Declines to run its own claim-extraction fan-out over the settings schema. When the claude-config-audit plugin is installed, routes the user to /claude-config-audit:settings-audit; otherwise states the request is out of scope.", + "files": [], + "expectations": [ + "Identifies settings.json / .mcp.json / hooks / permissions schema-drift as out of scope for this skill's claim-verification lane", + "When claude-config-audit is installed, routes the user to /claude-config-audit:settings-audit rather than a bare or differently-named skill", + "When claude-config-audit is not installed, states the request is out of scope instead of routing to a nonexistent skill", + "Does not run its own claim-extraction fan-out over the settings/MCP schema" + ] } ] } diff --git a/plugins/discovery/.claude-plugin/plugin.json b/plugins/discovery/.claude-plugin/plugin.json index 784ce7334..9ebe0acd9 100644 --- a/plugins/discovery/.claude-plugin/plugin.json +++ b/plugins/discovery/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "discovery", - "version": "0.4.1", + "version": "0.5.0", "description": "Structured discovery before changes: explore the local codebase (inline or in an isolated forked subagent) and run disciplined multi-source external research with source tiers, falsification, and recency gates — persisting EXPLORE.md / RESEARCH.md handoff artifacts.", "author": { "name": "Melodic Software", diff --git a/plugins/discovery/CHANGELOG.md b/plugins/discovery/CHANGELOG.md index fa61b35cf..dcbee204b 100644 --- a/plugins/discovery/CHANGELOG.md +++ b/plugins/discovery/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog — discovery plugin +## 0.5.0 — 2026-07-15 + +### Added + +- Research floor-scaling and broad-topic-minimums evals in `skills/research/evals/evals.json`: + `floor-scaling-single-product` pins that Phase 2 query count tracks the Phase 1 written gap + count rather than stopping at the 3-query floor (SKILL.md's "floor is a starting point, not a + target"); `broad-topic-triple-tool-comparison` pins that a 3-tool comparison topic fires the + doubled phase/query/source minimums (SKILL.md item 8, discipline.md's "Broad-topic + auto-detect"). + ## 0.4.0 — 2026-07-14 Adopt the marketplace topic-docs convention, contract v1.0.0 diff --git a/plugins/discovery/skills/research/evals/evals.json b/plugins/discovery/skills/research/evals/evals.json index 47bfc58a1..7e9c94484 100644 --- a/plugins/discovery/skills/research/evals/evals.json +++ b/plugins/discovery/skills/research/evals/evals.json @@ -73,6 +73,47 @@ "When blog consensus contradicts the primary source, the primary wins and the conflict is flagged explicitly", "No accepted claim is backed exclusively by Tier-2 secondary sources" ] + }, + { + "id": 7, + "name": "floor-scaling-single-product", + "prompt": "We're deciding whether to adopt a particular open-source in-process cache library for a new service. Before committing, research its actual production readiness: current stability/release maturity, adapter support across the data stores we use, history of breaking changes, integration story with our dependency-injection container, testing/mocking support, and any documented limitations.", + "expected_output": "Single-product topic — broad-topic auto-detect does not fire (one library named, no comparison), so standard floors apply. The written Phase 1 gap list surfaces well more than 3 open questions (release/stability status, adapter maturity, breaking-change history, DI integration, testing support, known limitations), and Phase 2 runs one query per gap rather than stopping at the 3-query floor. Includes the mandatory falsification query, primaries fetched directly, and the outcome gate before presenting.", + "files": [], + "expectations": [ + "Does not treat this as a broad-topic / comparison — only one library is named, so standard (non-doubled) minimums apply", + "Phase 1 launches 3+ parallel queries across 3+ source categories (official docs, upstream repo/changelog, AI-synthesis or web)", + "Emits a WRITTEN Phase 1 analysis block — numbered gaps + numbered conflicts + leading hypothesis — before composing any Phase 2 query", + "The written gap list has 5 or more numbered entries (e.g., release/stability status, adapter/connector maturity, breaking-change history, DI/framework integration, testing/mocking support, known limitations) — a production-readiness sweep surfaces well past the 3-query floor", + "Phase 2 query count matches or exceeds the run's own written gap count — it does NOT stop at 3 queries while numbered gaps remain open; the floor is treated as a starting point, not a target", + "Each Phase 2 query traces to a named entry in the written gap/conflict block — no padding queries beyond the gap surface either", + "Phase 2 includes exactly one falsification query against the Phase 1 leading hypothesis (e.g., searching for 'deprecated', 'known limitations', or a maintainer disclaimer contradicting the hypothesis)", + "Each claim's primary is fetched directly (the library's official docs / GitHub repo / changelog) this turn — not accepted from a synthesis or SERP summary", + "Recency gate is satisfied — the latest upstream changelog/release is fetched this turn and cross-checked against the claims", + "Runs the outcome gate before presenting — any claim below the 1-primary + 2-corroborator bar is reported as a Gap, not a finding", + "Includes a project-fit check against the consuming project's own conventions" + ] + }, + { + "id": 8, + "name": "broad-topic-triple-tool-comparison", + "prompt": "We're choosing among three competing message-queue libraries for a new service — Queue Tool A, Queue Tool B, and Queue Tool C. Research which is the better fit for our throughput and durability requirements.", + "expected_output": "Broad-topic auto-detect fires — 3 tools named in an explicit X vs Y vs Z comparison — so doubled minimums apply: 6+ queries per phase, 12+ total, 5+ distinct tool types across the topic, 4+ Tier 0/1 sources per claim. A written Phase 1 gap/conflict/hypothesis block drives Phase 2, which includes the mandatory falsification query. Phase 3 uses the tool-ecosystem fallback (Path B) since no preferred-source author covers these three placeholder tools. Outcome gate runs before presenting.", + "files": [], + "expectations": [ + "Detects the broad-topic trigger (3 distinct tools named, explicit comparison) and applies DOUBLED minimums — 6+ Phase 1 queries, 12+ total queries across all phases, 5+ distinct tool types, 4+ Tier 0/1 sources per accepted claim", + "Phase 1 launches 6+ queries (not the standard 3+) using 3+ distinct tool types", + "Emits a WRITTEN Phase 1 analysis block — leading hypothesis + numbered gaps + numbered conflicts + tool-diversity audit + recency status + falsification candidate — before composing any Phase 2 query", + "Phase 2 query count scales to the numbered gaps/conflicts from Phase 1, not a flat 6; each query traces to a named entry in the written block", + "Phase 2 includes exactly one mandatory falsification query attempting to break the Phase 1 leading hypothesis (e.g., 'Queue Tool A durability guarantees disputed' or 'Queue Tool B does NOT support clustering')", + "Phase 2 fetches each of the three tools' primary docs/changelog directly this turn rather than accepting a SERP or synthesis summary", + "Phase 3 fires the tool-ecosystem fallback (Path B) because no preferred-source roster names an author covering these three placeholder tools — cites official maintainer source, upstream changelog/releases, and one recognized industry authority for each", + "Evidence table tracks Tier + tool-diversity + confidence per claim; rejects any claim whose sources are all Tier 2 or all from one tool category", + "Source-tier ratio enforced — every accepted claim has >=1 Tier 0/1 source plus >=2 independent corroborators; three citations of the same secondary pool count as one source, not three", + "Confidence calibration applied — every comparison claim marked HIGH/MEDIUM/LOW; LOW-confidence claims are not used as the basis for a recommendation", + "Runs the outcome gate before presenting — all 8 binary criteria checked against the evidence table and written gap lists; reports pass or the failing criterion and what was re-run", + "Output names an explicit per-tool comparison conclusion, or explicitly reports insufficient evidence as a Gap, rather than picking a favorite without evidence backing" + ] } ] } diff --git a/plugins/docs-hygiene/.claude-plugin/plugin.json b/plugins/docs-hygiene/.claude-plugin/plugin.json index 6ea411f22..47e01f979 100644 --- a/plugins/docs-hygiene/.claude-plugin/plugin.json +++ b/plugins/docs-hygiene/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "docs-hygiene", - "version": "0.3.0", + "version": "0.4.0", "description": "Documentation-hygiene toolkit of five skills: compress (flavor-trim markdown with a semantic-diff safety net), declutter (classify markdown noise), extract-ssot (deduplicate repeated content into a single source of truth), encapsulation-audit (detect citations into skill-private surfaces), and rename-references (sweep stale references after renames).", "author": { "name": "Melodic Software", diff --git a/plugins/docs-hygiene/CHANGELOG.md b/plugins/docs-hygiene/CHANGELOG.md index 1e909a5e9..191728995 100644 --- a/plugins/docs-hygiene/CHANGELOG.md +++ b/plugins/docs-hygiene/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog — docs-hygiene plugin +## 0.4.0 — 2026-07-15 + +Added: + +- Self-contained, bundled eval fixtures: compress's `audit-classification-table` + case (`evals/fixtures/audit-fixture-dir/`) and declutter's + `opt-out-and-section-exemptions-respected` case + (`evals/fixtures/legit-optouts.md`) — both previously unfalsifiable prose + prompts referencing nonexistent files. +- The "add an eval case" clause, re-added to the two Gotchas/Recheck-trigger + bullets in rename-references/SKILL.md. + ## 0.3.0 — 2026-07-14 Adopt the marketplace topic-docs convention diff --git a/plugins/docs-hygiene/skills/compress/evals/evals.json b/plugins/docs-hygiene/skills/compress/evals/evals.json index 30f0886d9..02910f36b 100644 --- a/plugins/docs-hygiene/skills/compress/evals/evals.json +++ b/plugins/docs-hygiene/skills/compress/evals/evals.json @@ -16,14 +16,23 @@ }, { "id": 2, - "name": "audit-is-read-only", - "prompt": "/compress audit docs/", - "expected_output": "Runs the audit action only: a read-only dry-run that classifies each target SKIP / COMPRESS / UNCERTAIN with an expected-yield estimate. No file is edited and no semantic-diff subagent is dispatched.", - "files": [], + "name": "audit-classification-table", + "prompt": "Run /compress audit against evals/fixtures/audit-fixture-dir (relative to the skill directory) — a directory of three files: an author-time-disciplined reference doc, a verbose prose doc, and a mixed doc.", + "expected_output": "Runs the audit action only: a read-only dry-run using the author-time-signal heuristic in context/target-types.md. No file is edited and no semantic-diff subagent is dispatched. Output is a markdown table with columns target/expected_yield_pct/classify/reason, one row per file, sorted lexically. `lean.md` classifies SKIP (signal 4: explicit compression-discipline citation fires). `verbose.md` classifies COMPRESS (signal 5 fallback: verbose-prose baseline, no code-token or cross-reference density). `mixed.md` classifies UNCERTAIN or COMPRESS (signals 2/3 borderline on a small mixed-density file).", + "files": [ + "evals/fixtures/audit-fixture-dir/lean.md", + "evals/fixtures/audit-fixture-dir/verbose.md", + "evals/fixtures/audit-fixture-dir/mixed.md" + ], "expectations": [ "The run classifies targets as SKIP / COMPRESS / UNCERTAIN rather than editing them", "No file is modified (audit is read-only)", - "No semantic-diff subagent is dispatched (dispatch is default-action only, not audit)" + "No semantic-diff subagent is dispatched (dispatch is default-action only, not audit)", + "Output is a markdown table with exactly 3 data rows (one per fixture file), columns target/expected_yield_pct/classify/reason", + "`lean.md` classifies SKIP, citing the explicit compression-discipline citation (signal 4)", + "`verbose.md` classifies COMPRESS, citing the verbose-prose fallback (signal 5)", + "`mixed.md` classifies UNCERTAIN or COMPRESS", + "Rows sorted lexically by target (deterministic ordering)" ] }, { diff --git a/plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/lean.md b/plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/lean.md new file mode 100644 index 000000000..1e3d08b25 --- /dev/null +++ b/plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/lean.md @@ -0,0 +1,19 @@ +# Lean Reference + +Author-time-disciplined body. Verifiable claims with citations. Already follows +this repo's AGENTS.md "Prose compression discipline" convention. + +## Tier policy + +Tier 0 outranks Tier 1 outranks Tier 3. Promote Tier 3 claims before edit per +`.claude/rules/claim-verification.md`. + +## Verification matrix + +| Claim type | Verify via | +|---|---| +| CLI flag | ` --help` this turn | +| File path | `Glob` / `ls` this turn | +| Library API | Read source OR fetch official doc | + +Cost: ~1 tool call. No exceptions. diff --git a/plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/mixed.md b/plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/mixed.md new file mode 100644 index 000000000..32703ac01 --- /dev/null +++ b/plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/mixed.md @@ -0,0 +1,20 @@ +# Mixed Reference + +This file has a basically mixed shape — some really tight directives, some quite verbose filler. Used as fixture for batch-mode compression evals. + +## Hard rules + +- **Tier 0 outranks Tier 3.** Non-negotiable. +- **Verify CLI flags via ` --help` this turn.** Training-recall is Tier 3 and MUST be promoted. + +## Some Additional Context + +So basically, the point of this section is just to add some kind of verbose prose into the mix. You'll notice that it's really not all that disciplined — there's quite a lot of hedging and filler that could pretty easily be cut. + +## Output schema + +```text +: (pct=N.N%, lint=PASS|FAIL) +``` + +`action` ∈ {`compressed`, `reverted`, `skipped`}. diff --git a/plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/verbose.md b/plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/verbose.md new file mode 100644 index 000000000..89ee076c0 --- /dev/null +++ b/plugins/docs-hygiene/skills/compress/evals/fixtures/audit-fixture-dir/verbose.md @@ -0,0 +1,25 @@ +# A Pretty Verbose Reference Document + +This is really just a kind of document that has quite a lot of unnecessary hedging and filler in it. We're going to basically walk through several concepts here, but honestly most of the prose could be significantly tightened without any kind of loss of meaning whatsoever. + +## What This Document Is Actually About + +So, essentially, this document is just sort of meant to demonstrate what verbose, undisciplined prose looks like in practice. It's really not all that different from the kind of thing you might find in various onboarding documents or in third-party content that has been pasted into the repo without much in the way of careful editing. + +## A Few Important Points to Consider + +There are basically a few key points that you should probably keep in mind as you read through this particular document: + +- First of all, this is really just an example, so you shouldn't necessarily take any of the specific content here too seriously +- Second, the actual style of the prose is intentionally somewhat loose and conversational +- Third, there are quite a lot of words here that could pretty easily be cut without changing the overall meaning of the text in any meaningful way + +## Some Additional Context + +It's probably worth noting that this document is essentially being used as a fixture for the compress skill's evaluation suite. That basically means that we're using it to verify that the skill can actually recognize and remove flavor from verbose prose like this. + +If you happen to be reading this for any other reason, well, then you should probably just disregard pretty much everything you see here. It's really not meant to be taken as authoritative in any way, shape, or form. + +## A Final Note + +That's basically all we have to say. Hope this is somewhat useful as a demonstration! diff --git a/plugins/docs-hygiene/skills/declutter/evals/evals.json b/plugins/docs-hygiene/skills/declutter/evals/evals.json index 96dedd375..51f875fc0 100644 --- a/plugins/docs-hygiene/skills/declutter/evals/evals.json +++ b/plugins/docs-hygiene/skills/declutter/evals/evals.json @@ -82,6 +82,21 @@ "The `.claude/notes/` citation IS flagged as a ghost-ref despite the `` placeholder", "Output explains the location is retired, so the placeholder exemption does not apply to it" ] + }, + { + "id": 8, + "name": "opt-out-and-section-exemptions-respected", + "prompt": "Declutter this file: read evals/fixtures/legit-optouts.md relative to the skill directory and audit it for noise.", + "expected_output": "Reports zero Tier 1 / Tier 2 / Tier 3 findings. The `` wrapped paragraph and the `` wrapped line are both skipped entirely, and the `## ADR amendment block` (with its dated `### … amendment` subsections), `## Recheck triggers`, `## Cross-references`, `## Sources`, and `## History` sections are all respected as exempt per Section EXEMPTIONS.", + "files": ["evals/fixtures/legit-optouts.md"], + "expectations": [ + "Respects the `` paragraph-scope opt-out — does not flag the citation-shaped content in the wrapped paragraph", + "Respects the `` line-scope opt-out — does not flag the citation-shaped content on the wrapped line", + "Does not flag the `## ADR amendment block` heading or its dated `### … amendment` subsections (amendment-block exemption)", + "Does not flag `## Recheck triggers`, `## Cross-references`, `## Sources`, or `## History` sections (Section EXEMPTIONS)", + "Reports zero Tier 1 findings overall", + "Reports zero Tier 2 findings overall" + ] } ] } diff --git a/plugins/docs-hygiene/skills/declutter/evals/fixtures/legit-optouts.md b/plugins/docs-hygiene/skills/declutter/evals/fixtures/legit-optouts.md new file mode 100644 index 000000000..7ba7e089d --- /dev/null +++ b/plugins/docs-hygiene/skills/declutter/evals/fixtures/legit-optouts.md @@ -0,0 +1,49 @@ +# Bar Convention + +Defines the Bar convention for bar-related rule files. + +## Convention body + +The Bar convention requires every Bar-producing function to honor the BarContract interface. + + +Empirically observed 2026-04-22 on slice `bar-rollout`: skipping the BarContract step caused a regression in 3 downstream consumers. This narrative is legitimately inline because the incident drove the convention's existence and rewriting it as a footer would lose the prose-narrative shape that makes the rule's motivation legible. The opt-out marker above wraps this paragraph per the markdown-discipline.md opt-out convention. + +The following paragraph carries no opt-out wrap and demonstrates a clean directive. + +Bar values MUST be immutable post-construction. Mutability breaks downstream invariants. + + +Empirically observed 2026-05-01: this single citation line is opt-out-wrapped for teaching purposes. + +## ADR amendment block + +### 2026-04-30 amendment + +Status amended from `proposed` to `accepted`. Bar contract finalized after stakeholder review. + +### 2026-05-15 amendment + +Status amended from `accepted` to `superseded by ADR-NNNN`. Bar replaced by Baz per ADR-NNNN. + +## Recheck triggers + +| Condition | Action | +|---|---| +| Bar consumer count exceeds 10 | Evaluate whether BarContract should be split | +| Downstream framework drops support for the BarContract pattern | Reconcile or migrate to successor pattern | + +## Cross-references + +- `baz-conventions.md` — Bar's successor convention per ADR-NNNN +- `review/architecture.md` "Dependency direction" — Bar inherits the layer rules + +## Sources + +- [Bar pattern in upstream framework](https://example.invalid/docs/bar) — original Bar shape this convention inherits +- `incident-2026-04-22.md` — incident driving the BarContract requirement (promoted from a retired slice per Promotion paths — `git log -- .work/bar-rollout/`) + +## History + +- 2026-04-30 — Bar contract finalized after stakeholder review +- 2026-05-15 — Bar superseded by Baz per ADR-NNNN diff --git a/plugins/docs-hygiene/skills/rename-references/SKILL.md b/plugins/docs-hygiene/skills/rename-references/SKILL.md index 0273022e1..b9ae8d480 100644 --- a/plugins/docs-hygiene/skills/rename-references/SKILL.md +++ b/plugins/docs-hygiene/skills/rename-references/SKILL.md @@ -147,7 +147,7 @@ Paths skipped from sweeps automatically: - **Ambiguous bucket is mandatory triage, not optional.** English-verb collisions are the highest false-positive vector. If a token is in the blocklist, force into ambiguous regardless of position. Cost of one extra confirmation prompt is far lower than silently mangling prose. - **Re-sweep until count == 0.** Don't trust Phase 5 ended cleanly without verification. Phase 6 is the gate. - **Plan-doc exclusion is mandatory.** The active plan/work-notes document *documents the rename* and contains both old and new names by design. Editing it would break the documentation narrative. -- **Pattern library evolves.** When Phase 6 finds a NEW form, treat as a learning event: extend `context/patterns.md`. Future renames benefit immediately. +- **Pattern library evolves.** When Phase 6 finds a NEW form, treat as a learning event: extend `context/patterns.md`, add an eval case. Future renames benefit immediately. - **A file MOVE breaks the moved files' own relative paths — sweep INSIDE the moved set, not just refs TO it.** When `git mv` changes directory depth, relative refs *inside* the moved files (`source ../../lib.sh`, `# shellcheck source=../../../../tests/...`, relative markdown links) silently break — they carry no renamed token, so every token-keyed pattern returns clean while the moved file itself is broken. After any depth-changing move: `grep -nE '\.\./' ` + re-run the moved code from its new location (tests, `--help`). Real example: a directory promotion left a `# shellcheck source=` directive pointing four levels up when the new home was two. - **A rename couples sibling renames — sweep each as its own pair.** Renaming a skill or identifier usually drags coupled siblings that do NOT contain the primary token: dot-form action/mode IDs (`verify.runtime-affecting-paths` — Form 12), internal mode names (`quality` mode), content-file basenames (context/quality.md style paths). A phase-scoped, skill-only grep on the primary token (`/verify`) leaves these EXTERNAL refs — in skill bodies, config files, and other skills' dispatch tables — unverified. A slash-anchored token sweep can return "clean" while `.id` / `` / `.md`-path refs survive elsewhere. Before declaring a rename complete: enumerate the coupled identifiers (Survey phase) and run a sweep per pair. @@ -163,4 +163,4 @@ Paths skipped from sweeps automatically: | `git mv` just executed | Invoke `/rename-references audit` to surface stragglers | | Pre-PR: working tree contains R-status files | Suggest `/rename-references audit` before final verification | | `/rename-references` finds 0 matches | Proceed to verification (or done if already past it) | -| `/rename-references` finds NEW form not in pattern library | Update `context/patterns.md`, re-iterate | +| `/rename-references` finds NEW form not in pattern library | Update `context/patterns.md`, add eval case, re-iterate | diff --git a/plugins/event-storming/.claude-plugin/plugin.json b/plugins/event-storming/.claude-plugin/plugin.json index 49101e256..67f873b67 100644 --- a/plugins/event-storming/.claude-plugin/plugin.json +++ b/plugins/event-storming/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "event-storming", - "version": "0.3.0", + "version": "0.4.0", "description": "EventStorming for domain discovery — a methodology skill (Big Picture / Process Modeling / Design-Level facilitation reference, notation, patterns) and a simulation skill (agentic multi-persona workshops that produce a structured-markdown model by default; a live Miro-board rendering path is available when the first-party miro plugin is enabled).", "author": { "name": "Melodic Software", diff --git a/plugins/event-storming/CHANGELOG.md b/plugins/event-storming/CHANGELOG.md index c56f8f8b5..116bf1857 100644 --- a/plugins/event-storming/CHANGELOG.md +++ b/plugins/event-storming/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to the `event-storming` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.4.0] + +### Added + +- `--discover-bcs` eval + fixture for the offline/exported-board path: a completed Big Picture board's items supplied directly as an export (`evals/fixtures/big-picture-board-export.md`) run through Bounded Context Discovery mechanically, without requiring a Miro connection. Distinct from the existing board-URL scenario, which still requires Miro to read a live board. + ## [0.3.0] ### Added diff --git a/plugins/event-storming/skills/simulation/evals/evals.json b/plugins/event-storming/skills/simulation/evals/evals.json index 9e89c8ed9..f1290c58a 100644 --- a/plugins/event-storming/skills/simulation/evals/evals.json +++ b/plugins/event-storming/skills/simulation/evals/evals.json @@ -85,6 +85,23 @@ "On finding no prior Process Modeling board, surfaces the missing prerequisite (offering to run --process-model first) instead of fabricating a process model or aggregates from scratch", "Routes to Design-Level behavior (e.g. Blank Aggregates), not to Big Picture or the no-args interactive discovery flow" ] + }, + { + "id": 8, + "name": "discover-bcs-offline-export-mechanical-output", + "prompt": "--discover-bcs — no Miro MCP server is available this session, and no board URL is being given. Instead, treat evals/fixtures/big-picture-board-export.md (relative to the skill directory) as an already-exported set of board items from a completed Big Picture board (People & Systems and Walk-through phases done). Run bounded-context discovery against that export.", + "expected_output": "This is not the read-a-live-board-by-URL scenario: no board URL was given, and the caller already supplied the exported board items directly as input. Because Bounded Context Discovery applies Brandolini's 6 heuristics mechanically against parsed board data rather than by making live Miro calls itself, the skill runs the full analysis directly against the supplied export and produces an architect's post-workshop BC table (not a workshop output), with a per-heuristic evidence citation per row, WITHOUT requiring or requesting a Miro connection. It should surface the 'Enrollment' language divergence, the parallel authoring/learning zones, and the phase splits at the pivotal events; use short 2-3 word BC names; and tie the arrow-voting winner (refund disputes) to a recommended next-exploration BC.", + "files": ["evals/fixtures/big-picture-board-export.md"], + "expectations": [ + "Does not ask to connect Miro or claim it cannot proceed without a live board connection — the supplied export substitutes for a live board read", + "Applies Brandolini's 6 heuristics (H1 business phases, H2 swimlanes, H3/H4 people/hovering, H5 hot spots/divergence, H6 language divergence) to the exported board data", + "Produces a bounded-context table that includes a per-heuristic evidence column giving traceability for each BC", + "Frames bounded-context discovery as the architect's post-workshop homework, NOT as a workshop phase or participant output", + "Names bounded contexts with short 2-3 word names", + "Identifies the '\"Enrollment\" means two different things' divergence (platform signup vs course access) as a context boundary signal", + "Cross-references the arrow-voting winner (refund disputes) to recommend which BC to explore next", + "Does not contradict the board-URL scenario (eval 3): a live Miro board reached by URL still requires Miro to read, but a directly-supplied export does not" + ] } ] } diff --git a/plugins/event-storming/skills/simulation/evals/fixtures/big-picture-board-export.md b/plugins/event-storming/skills/simulation/evals/fixtures/big-picture-board-export.md new file mode 100644 index 000000000..b3bb49727 --- /dev/null +++ b/plugins/event-storming/skills/simulation/evals/fixtures/big-picture-board-export.md @@ -0,0 +1,74 @@ +# Big Picture board export — "Online Course Marketplace" + +Text export of a completed Big Picture EventStorming board (People & Systems and +Walk-through phases done). Use this as the board data for a `--discover-bcs` run — +run Brandolini's 6 heuristics against it. No live Miro read is required; treat the +rows below as the parsed board items. + +## Timeline (orange domain events, left → right), grouped by timeline zone + +### Zone 1 — Onboarding & content + +- `Instructor Signed Up` — persona: Instructor — y-row: instructor +- `Instructor Profile Verified` — persona: Support Agent — y-row: support +- `Course Draft Created` — persona: Instructor — y-row: instructor +- `Course Content Uploaded` — persona: Instructor — y-row: instructor +- `Course Submitted For Review` — persona: Instructor — y-row: instructor +- `Course Approved` — persona: Support Agent — y-row: support +- `Course Published` — persona: Instructor — y-row: instructor + - NOTE: Support Agent's sticky for this same moment reads `Course Went Live` + +`--- PIVOTAL: Course Published ---` + +### Zone 2 — Discovery & purchase + +- `Student Signed Up` — persona: Student — y-row: student + - NOTE: Instructor referred to this moment as `Student Enrolled` (see divergence below) +- `Course Added To Cart` — persona: Student — y-row: student +- `Checkout Started` — persona: Student — y-row: student +- `Payment Authorized` — persona: Finance — y-row: finance — external: Payment Gateway +- `Payment Captured` — persona: Finance — y-row: finance — external: Payment Gateway +- `Enrollment Granted` — persona: Student — y-row: student + - NOTE: this is enrollment INTO A COURSE, distinct from `Student Signed Up` + +`--- PIVOTAL: Payment Captured ---` + +### Zone 3 — Learning & completion (runs in parallel with Zone 1 authoring — different timescale) + +- `Lesson Started` — persona: Student — y-row: student +- `Lesson Completed` — persona: Student — y-row: student +- `Quiz Passed` — persona: Student — y-row: student +- `Course Completed` — persona: Student — y-row: student +- `Certificate Issued` — persona: Student — y-row: student — external: Email Service + +### Zone 4 — Money & disputes + +- `Refund Requested` — persona: Student — y-row: student + - NOTE: Finance's sticky for this same moment reads `Chargeback Filed` +- `Refund Reviewed` — persona: Support Agent — y-row: support +- `Refund Issued` — persona: Finance — y-row: finance — external: Payment Gateway +- `Instructor Payout Calculated` — persona: Finance — y-row: finance +- `Instructor Payout Sent` — persona: Finance — y-row: finance — external: Payment Gateway + +## People (small yellow) + +- Instructor — authors and publishes courses +- Student — discovers, buys, learns +- Support Agent — reviews courses, mediates refunds +- Finance — handles payments, refunds, payouts + +## External systems (pink) + +- Payment Gateway — authorizes, captures, refunds, pays out +- Email Service — sends certificates and notifications + +## Hot spots (magenta) + +- `[DIVERGENCE] "Enrollment"` — Student uses it for platform signup (`Student Signed Up`); + Instructor/Finance use it for course access (`Enrollment Granted`). Same word, two meanings. +- `Refund disputes take 6+ days` — Support and Finance both touch refunds; nobody owns the SLA. +- `Who approves a course — Support or an editor?` — approval ownership unclear. + +## Arrow voting + +- Winner (most votes): `Refund disputes take 6+ days` diff --git a/plugins/review-toolkit/.claude-plugin/plugin.json b/plugins/review-toolkit/.claude-plugin/plugin.json index cd29bad37..eb22cb98a 100644 --- a/plugins/review-toolkit/.claude-plugin/plugin.json +++ b/plugins/review-toolkit/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "review-toolkit", - "version": "0.5.0", + "version": "0.6.0", "description": "Code-review toolkit: six read-only reviewer agents (code, security, architecture, doc drift, build/test/lint, CI-log audit) plus two orchestration skills — a single-lens quality gate and a multi-surface review fan-out with severity-ranked, deduplicated findings.", "author": { "name": "Melodic Software", diff --git a/plugins/review-toolkit/CHANGELOG.md b/plugins/review-toolkit/CHANGELOG.md index 87d783844..7c5eff03d 100644 --- a/plugins/review-toolkit/CHANGELOG.md +++ b/plugins/review-toolkit/CHANGELOG.md @@ -3,6 +3,20 @@ All notable changes to the `review-toolkit` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.6.0] + +### Added + +- **Restored fanout regression evals.** `code-review-fanout`'s `evals/evals.json` gains 14 cases + (ids 7–20) covering behavior that was still documented but had lost eval coverage: dedup and + severity-derivation (Stage 3/4 cross-surface merge, content-derived severity for + no-native-severity surfaces), the fix-pass safety fence (correctness findings are never routed + to `/simplify`, branch-scoped findings lookup, mixed-class routing), and run-everything's + null-reconciliation and priority-ordering (named null leaves, the tier-1 barrier ahead of + tier-2). Also restored: per-tier surface routing and promotion, the large-tier ownerless-slice + exclusions, the findings-file shape contract, the clean-tree short-circuit, and graceful + orchestrator-absent degradation. + ## [0.5.0] ### Added diff --git a/plugins/review-toolkit/skills/code-review-fanout/evals/evals.json b/plugins/review-toolkit/skills/code-review-fanout/evals/evals.json index fc43def4e..dc5e94ce0 100644 --- a/plugins/review-toolkit/skills/code-review-fanout/evals/evals.json +++ b/plugins/review-toolkit/skills/code-review-fanout/evals/evals.json @@ -72,6 +72,177 @@ "Output names the skipped PR-mutation surface in its surfaces list rather than silently omitting it", "Output keeps the review report-only, posting nothing to the open PR absent an explicit opt-in" ] + }, + { + "id": 7, + "name": "tier-small-leaf-agents-only", + "prompt": "Fan out a review of this small 30-line change — nothing security-sensitive, no layer boundaries crossed.", + "expected_output": "The small tier dispatches code-reviewer only (plus security-reviewer solely when a security-sensitive path is touched) and does not promote to medium or large.", + "files": ["a 30-line non-security, non-layer-boundary change staged"], + "expectations": [ + "Classifies the change as tier small (<50 changed lines) and does not promote to medium", + "Dispatches code-reviewer only; does NOT dispatch architecture-guardian or any orchestrator plugin at this tier", + "Names security-reviewer as SKIPPED in the tier-transparency line since no auth/input/secrets path was touched" + ] + }, + { + "id": 8, + "name": "tier-small-security-promotes-to-medium", + "prompt": "Fan out a review of this 25-line change — it touches an authentication check.", + "expected_output": "A small-sized diff that is security-sensitive promotes to the full medium-tier surface set rather than only adding security-reviewer to the small set.", + "files": ["a 25-line change touching an auth/identity path"], + "expectations": [ + "Detects the security-sensitive path and promotes the diff from small to medium tier despite the line count", + "Dispatches the full medium-tier surface set (code-reviewer, security-reviewer, orchestrator plugin(s), and architecture-guardian when structural paths are touched) rather than only adding security-reviewer to the small set", + "The tier-transparency line names the promotion trigger rather than silently reporting a small-tier classification" + ] + }, + { + "id": 9, + "name": "large-tier-ownerless-slices-and-exclusions", + "prompt": "Fan out a review of this 400-line change spanning multiple directories and ecosystems.", + "expected_output": "The large tier dispatches the medium set plus the project's ownerless review-criteria docs as slice-subagents, while excluding agent-owned criteria and the two non-diff-review agents.", + "files": ["a >300-line cross-cutting change across multiple directories"], + "expectations": [ + "Classifies tier as large (>300 lines or cross-cutting) and dispatches the medium set plus discovered ownerless review-criteria slices", + "Does NOT dispatch code-quality/security/architecture criteria docs as separate slice-subagents — those are agent-owned, so re-reviewing them would be pure waste", + "Does NOT include ecosystem-specialist or ci-log-auditor in the roster — neither is a finding-producing diff reviewer", + "Orchestrator plugin(s) still run on the main thread, never inside a Workflow" + ] + }, + { + "id": 10, + "name": "dedup-cross-surface-max-severity", + "prompt": "Fan out a review where code-reviewer and security-reviewer both flag the same file:line.", + "expected_output": "Stage 3 dedup merges the same-issue findings from two surfaces into one ranked finding keyed by file plus line-proximity, carrying MAX severity and MAX confidence and listing both surfaces.", + "files": ["a change where two surfaces both flag the same file:line"], + "expectations": [ + "Merges the two same-issue findings into ONE record keyed by normalized file path + line-proximity bucket (±3 lines), not by category", + "Merged severity is the MAX of the contributing surfaces' severities, never a filtered-down value", + "The merged finding's surface list names BOTH contributing surfaces as a positive-agreement signal" + ] + }, + { + "id": 11, + "name": "plugin-findings-severity-derived-not-invented", + "prompt": "Fan out a review where the code-review orchestrator plugin returns confidence-filtered issues with no native severity.", + "expected_output": "The severity crosswalk derives a tier from finding content for surfaces with no native severity; a high confidence score is never treated as a severity tier.", + "files": ["a PR-present change where the code-review plugin emits confidence-filtered issues"], + "expectations": [ + "Findings from a surface with no native severity get a tier DERIVED from content (bug/correctness vs. convention-adherence), not copied from its confidence score", + "A high (>=80) confidence-filtered score maps to the confidence enum 'high', NOT to a high severity tier", + "Ambiguous derived-severity findings are tagged pending: human-tier rather than silently assigned a guessed tier" + ] + }, + { + "id": 12, + "name": "findings-file-stable-shape-relative-paths", + "prompt": "Fan out a review and persist the findings.", + "expected_output": "The persisted findings file has the stable frontmatter-plus-table-plus-appendix shape and cites file:line only as repo-relative paths.", + "files": ["any reviewable change"], + "expectations": [ + "Findings file frontmatter declares type: review-findings, plus date, branch, and tier", + "Body includes the ranked Findings table, a ## Unparsed appendix, and a ## Surfaces reconciliation line", + "All file:line citations in the findings are repo-relative — machine-absolute paths are relativized before writing" + ] + }, + { + "id": 13, + "name": "clean-tree-no-pr-spawns-nothing", + "prompt": "Fan out a review — my branch is clean and has no open PR.", + "expected_output": "A truly clean tree (no diff, not ahead of base, no open PR) reports that there is nothing to review and spawns nothing.", + "files": ["empty git diff vs. merge-base, branch not ahead of base, no open PR"], + "expectations": [ + "Reports that there are no changes to review", + "Spawns no reviewer surfaces", + "Writes no findings file" + ] + }, + { + "id": 14, + "name": "orchestrator-absent-degrades-gracefully", + "prompt": "Fan out a medium-tier review — none of the optional orchestrator plugins are installed.", + "expected_output": "An absent orchestrator plugin degrades to leaf-only coverage with an explicit skip notice; normalization still runs over whatever surfaces produced output.", + "files": ["a medium-tier change with no orchestrator plugin installed"], + "expectations": [ + "Dispatches only this plugin's leaf agents (and slice-subagents where applicable) when an orchestrator plugin is absent", + "Emits an explicit notice naming that orchestrator breadth was skipped rather than silently omitting it", + "Normalization and findings persistence still run over the surfaces that did produce output" + ] + }, + { + "id": 15, + "name": "run-everything-workflows-disabled-degraded-block", + "prompt": "/review-toolkit:code-review-fanout run-everything", + "expected_output": "The pre-launch gate detects Workflows are disabled and falls back to the main-thread coverage-parity roster with a DEGRADED block prepended, rather than attempting a launch.", + "files": ["CLAUDE_CODE_DISABLE_WORKFLOWS=1 set in the environment; a reviewable change"], + "expectations": [ + "The pre-launch availability gate detects the disabled state BEFORE attempting any Workflow launch", + "Falls back to spawning the same leaf roster on the main thread — findings coverage stays full", + "Prepends a DEGRADED block at the top of both the chat output and the persisted file naming the dropped execution properties, not any dropped findings" + ] + }, + { + "id": 16, + "name": "run-everything-null-leaf-named-not-dropped", + "prompt": "/review-toolkit:code-review-fanout run-everything", + "expected_output": "A leaf that returns null (budget exhaustion, a Bash failure, etc.) is named under a 'Returned no result' line, never silently absorbed into the roster count.", + "files": ["a run where one leaf returns null"], + "expectations": [ + "Reconciles the expected roster (ran) against the non-null returns rather than reporting only the non-null count", + "Names the null leaf explicitly under the ## Surfaces 'Returned no result' line, with cause when known", + "Does not silently cap or hide the null count — it is visible in the persisted report regardless of cause" + ] + }, + { + "id": 17, + "name": "run-everything-priority-ordering-tier1-barrier", + "prompt": "/review-toolkit:code-review-fanout run-everything", + "expected_output": "Tier-1 high-value reviewers (security-reviewer, architecture-guardian, code-reviewer) run as a barrier before tier-2 leaves, so budget exhaustion drops tier-2, never tier-1.", + "files": ["a run under a finite user turn-token budget that exhausts partway through the leaf fan-out"], + "expectations": [ + "security-reviewer, architecture-guardian, and code-reviewer run as tier 1, awaited as a barrier before any tier-2 leaf starts", + "When the budget exhausts during tier 2, those calls resolve to null rather than rejecting or aborting the whole run", + "Tier 1's results are unaffected by a tier-2 budget exhaustion — priority ordering protects the highest-value surfaces" + ] + }, + { + "id": 18, + "name": "fix-pass-splits-mixed-class-findings", + "prompt": "/review-toolkit:code-review-fanout fix", + "expected_output": "Fix-pass classifies findings by class and routes cleanup-class findings to /simplify while applying correctness-class findings via sequential scope-fenced fixes.", + "files": ["a current-branch findings file with both a cleanup-class (extract-helper) finding and a correctness-class (CRITICAL security) finding"], + "expectations": [ + "Locates the newest findings file for the CURRENT branch only", + "Classifies the cleanup-class finding and routes it to the bundled /simplify skill", + "Classifies the correctness-class finding and applies it via a sequential scope-fenced fix, not via /simplify", + "Emits the classification plan and confirms before mutating the working tree in an interactive session" + ] + }, + { + "id": 19, + "name": "fix-pass-no-findings-branch-scoped", + "prompt": "/review-toolkit:code-review-fanout fix", + "expected_output": "Fix-pass with no findings file for the current branch reports cleanly and stops, without scanning another branch's findings.", + "files": ["the current branch has no findings file; a different branch's findings directory does have one"], + "expectations": [ + "Reports 'No findings for branch ' and stops", + "Does NOT scan, read, or apply another branch's findings file — the branch check against frontmatter is the safety fence", + "Mutates nothing; suggests running the review first" + ] + }, + { + "id": 20, + "name": "fix-pass-correctness-never-simplify", + "prompt": "/review-toolkit:code-review-fanout fix", + "expected_output": "All correctness findings are applied via sequential scope-fenced fixes; /simplify is never invoked for a correctness-class finding.", + "files": ["a findings file whose only entries are correctness-class (a logic bug and a race condition)"], + "expectations": [ + "Applies both correctness findings via sequential, non-parallel scope-fenced fixes — one file at a time", + "Never invokes /simplify for a correctness-class finding — /simplify is cleanup-only and would silently drop the bug", + "Zero cleanup-class findings means /simplify is skipped entirely, not invoked 'to tidy anyway'", + "A low-confidence or high-blast-radius correctness finding is surfaced for the user's decision rather than auto-applied" + ] } ] } diff --git a/plugins/songwriting/.claude-plugin/plugin.json b/plugins/songwriting/.claude-plugin/plugin.json index 9e2463981..e4020dfe7 100644 --- a/plugins/songwriting/.claude-plugin/plugin.json +++ b/plugins/songwriting/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "songwriting", - "version": "0.3.1", + "version": "0.4.0", "description": "Songwriting craft companion — eight concern-scoped lyric-craft skills (workflow router, rhyme, object-writing, meter-prosody, song-form, co-write, diagnosis, daily-practice) applying Pat Pattison's methods, plus Suno v5.5 prompt engineering (style prompts, tagged lyrics, genre templates, troubleshooting).", "author": { "name": "Melodic Software", diff --git a/plugins/songwriting/CHANGELOG.md b/plugins/songwriting/CHANGELOG.md index 8f3b3020e..3f5e51efa 100644 --- a/plugins/songwriting/CHANGELOG.md +++ b/plugins/songwriting/CHANGELOG.md @@ -3,6 +3,20 @@ All notable changes to the `songwriting` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.4.0] + +### Added + +- **Behavioral evals restored, adapted to the multi-skill split.** The `pat-pattison` mega-skill's + full eval suite (13 cases) shipped zero replacement coverage when it decomposed in 0.2.0. All 13 + cases are ported forward, each adapted to the concern skill and action that now owns its behavior: + `workflow` (brainstorm, idea, fragment — 3 cases), `diagnosis` (demo, audit, variations — 3 cases), + `rhyme` (rhyme, datamuse — 2 cases), `song-form` (box-model, bridge — 2 cases), `co-write` + (title-game, co-write — 2 cases), and `object-writing` (metaphor-recipe — 1 case). No case was + dropped — every behavior the original suite exercised still exists in the split. Prompts and + expectations are updated to the plugin's `/songwriting: ` invocation form and + current `SKILL.md` contracts. + ## [0.3.0] ### Added diff --git a/plugins/songwriting/skills/co-write/evals/evals.json b/plugins/songwriting/skills/co-write/evals/evals.json new file mode 100644 index 000000000..038157a14 --- /dev/null +++ b/plugins/songwriting/skills/co-write/evals/evals.json @@ -0,0 +1,34 @@ +{ + "skill_name": "co-write", + "evals": [ + { + "id": 1, + "name": "title-game-cascade", + "prompt": "/songwriting:co-write title-game run me the Title Game starting from \"last call\"", + "expected_output": "Routes to the title-game action. Loads title-game.md, templates/title-game-prompt.md, and co-writing.md. Confirms cascade constraints (vowel rule, stress count, POV, genre, duration) before starting, chains 10-15 titles derived from the prior title's stressed vowel, mines the strongest 2-3 at the end, and names a next step for developing the chosen title.", + "files": [], + "expectations": [ + "Loads title-game.md via the `title-game` action", + "Sets or proposes cascade constraints before starting", + "Identifies the stressed vowel of each generated title", + "Generates 10-15 chained titles", + "Mines the strongest 2-3 at the end", + "Names a next step for developing the chosen title (e.g. `co-write title` or `workflow idea`) rather than stopping at the raw list" + ] + }, + { + "id": 2, + "name": "co-write-protocol-with-title-game", + "prompt": "/songwriting:co-write co-write session starting tonight. how do I open it with my writing partner?", + "expected_output": "Routes to the co-write (default) action. Loads co-writing.md, templates/co-write-session-opener.md, and process.md. Reads aloud the No-Free-Zone rules, recommends the Title Game as a warmup, proposes a session structure (open, seed, central section, trade verses, cliche check, revise solo), and does not skip the No-Free-Zone read.", + "files": [], + "expectations": [ + "Loads co-writing.md and references the No-Free-Zone protocol", + "Includes the No-Free-Zone rules (read aloud)", + "Recommends the Title Game as warmup OR an equivalent shared-seed exercise", + "Recommends agreement on the central section before drafting outward", + "Includes a 'revise solo later' follow-up step" + ] + } + ] +} diff --git a/plugins/songwriting/skills/diagnosis/evals/evals.json b/plugins/songwriting/skills/diagnosis/evals/evals.json new file mode 100644 index 000000000..979354c37 --- /dev/null +++ b/plugins/songwriting/skills/diagnosis/evals/evals.json @@ -0,0 +1,47 @@ +{ + "skill_name": "diagnosis", + "evals": [ + { + "id": 1, + "name": "demo-review-at-stage", + "prompt": "/songwriting:diagnosis demo \"\"", + "expected_output": "Routes to the demo action. Loads demo-review.md and templates/demo-review-prompt.md. Detects the completion stage (chorus only) and runs a stage-appropriate review rather than the full near-complete `diagnose` pass. Identifies the dominant problem — likely abstract telling or cliche — and proposes ONE focused next move, listing secondary observations briefly without fixing them.", + "files": [], + "expectations": [ + "Detects the completion stage as chorus-only (or similar partial state)", + "Runs the stage-appropriate demo review, NOT the full near-complete `diagnose` action", + "Surfaces ONE dominant next move", + "Lists secondary observations briefly (one line each) without fixing them", + "Names the response-filter Critique filter and Pre-lock filter status before emission" + ] + }, + { + "id": 2, + "name": "audit-checklist-pre-lock", + "prompt": "/songwriting:diagnosis audit \"midnight pulled the streetlight down and threw it in the river\"", + "expected_output": "Routes to the audit action. Loads audit-checklist.md and templates/audit-checklist-prompt.md. Walks the per-line audit clusters in order, presenting each as a deliberate choice point (pass/fail/skip-with-reason) rather than a pass/fail gate, and surfaces the first unchecked or dominant unresolved item.", + "files": [], + "expectations": [ + "Loads audit-checklist.md via the `audit` action", + "Walks multiple per-line audit clusters (e.g. stress/meter, rhyme, image, cliche, POV, prosody)", + "Presents each box as a deliberate choice point (tool, not a rule)", + "Surfaces the first unchecked OR dominant unresolved item", + "Does NOT block on any box — names the skip option when a box is skipped" + ] + }, + { + "id": 3, + "name": "variations-labeled", + "prompt": "/songwriting:diagnosis variations \"I keep walking past your door at night\" — 5 versions, different POV, different image", + "expected_output": "Routes to the variations action. Loads variations.md and templates/variations-prompt.md. Generates 4-6 labeled variations, each with an explicit axis label (POV / image / vowel / stress / rhyme-type / tone) and a gains/loses trade-off, without picking a winner.", + "files": [], + "expectations": [ + "Generates 4-6 variations", + "Each variation is explicitly labeled with an axis + shift (e.g. 'POV: 1st to 2nd')", + "Each variation states a gains/loses trade-off", + "Does NOT recommend a single best variation", + "If the writer asks for the 'best' one, pushes back asking for the load-bearing constraint" + ] + } + ] +} diff --git a/plugins/songwriting/skills/object-writing/evals/evals.json b/plugins/songwriting/skills/object-writing/evals/evals.json new file mode 100644 index 000000000..60dcb270b --- /dev/null +++ b/plugins/songwriting/skills/object-writing/evals/evals.json @@ -0,0 +1,19 @@ +{ + "skill_name": "object-writing", + "evals": [ + { + "id": 1, + "name": "metaphor-recipe-for-emotion", + "prompt": "/songwriting:object-writing metaphor-recipe I need a metaphor for grief that isn't a storm or an ocean", + "expected_output": "Routes to the metaphor-recipe action. Loads metaphor.md (the 8 named recipes) and templates/metaphor-recipe-prompt.md. Generates 6-8 metaphor candidates avoiding the named cliche families (storm-anger, fire-passion, ocean-sadness) per the user's constraint, labeling each with its type (expressed identity / qualifying / verbal) and a grounding suggestion.", + "files": [], + "expectations": [ + "Loads metaphor.md via the `metaphor-recipe` action", + "Avoids storm and ocean (named cliche families) per the user's constraint", + "Generates 6-8 candidate metaphors", + "Labels each with a metaphor type (expressed identity / qualifying / verbal)", + "Offers options rather than a single winning metaphor" + ] + } + ] +} diff --git a/plugins/songwriting/skills/rhyme/evals/evals.json b/plugins/songwriting/skills/rhyme/evals/evals.json new file mode 100644 index 000000000..b93a12079 --- /dev/null +++ b/plugins/songwriting/skills/rhyme/evals/evals.json @@ -0,0 +1,33 @@ +{ + "skill_name": "rhyme", + "evals": [ + { + "id": 1, + "name": "non-cliche-rhymes-internal-primary", + "prompt": "/songwriting:rhyme find rhymes for 'lonely' that aren't cliche", + "expected_output": "Loads rhyme-generation.md FIRST as the primary source. Applies the internal discipline: anchors the stressed vowel (long-O), runs the identity check (rejects 'only' as identity, not rhyme, since the pre-vowel consonant doesn't differ), walks the stability-tier scale, pulls from the phonetic family, and flags cliche partners (e.g. slowly/holy). Surfaces 8-15 candidates across tiers with labels. Does not lead with Datamuse.", + "files": [], + "expectations": [ + "Loads rhyme-generation.md before any external API call", + "Identifies the stressed vowel as long-O", + "Rejects 'only' as identity (no pre-vowel consonant difference)", + "Flags cliche pairs (e.g. holy/lonely, slowly/lonely)", + "Surfaces 8-15 candidates with stability-tier and cliche-risk labels", + "If Datamuse is used, it is labeled as a supplement to internal generation, not the primary source" + ] + }, + { + "id": 2, + "name": "syllable-count-via-datamuse", + "prompt": "/songwriting:rhyme datamuse how many syllables is 'disestablishmentarianism'?", + "expected_output": "Recognizes this as a polysyllabic / rare word where internal counting is unreliable. Routes to the datamuse action, loading ai-tools.md and invoking scripts/datamuse.sh for syllable lookup, returning an exact count rather than a guess.", + "files": [], + "expectations": [ + "Acknowledges this is a polysyllabic / rare word", + "Routes to the `datamuse` action (ai-tools.md + scripts/datamuse.sh) for syllable verification", + "Returns an exact syllable count, not a guess", + "If the script is unavailable (no bash/curl/jq), degrades gracefully and says so rather than fabricating a count" + ] + } + ] +} diff --git a/plugins/songwriting/skills/song-form/evals/evals.json b/plugins/songwriting/skills/song-form/evals/evals.json new file mode 100644 index 000000000..827a977e2 --- /dev/null +++ b/plugins/songwriting/skills/song-form/evals/evals.json @@ -0,0 +1,32 @@ +{ + "skill_name": "song-form", + "evals": [ + { + "id": 1, + "name": "second-verse-stagnation-box-model", + "prompt": "/songwriting:song-form box-model my second verse repeats the first — same scene, same speaker, same time. how do I fix it?", + "expected_output": "Routes to the box-model action. Loads box-model.md and verse-development.md. Names the travelogue diagnosis and proposes a division-of-labor formula (You-I-We or Past-Present-Future or another axis), surfacing what verse 2 should DO that verse 1 doesn't, and mentions a 'what came before' diagnostic or equivalent reframing.", + "files": [], + "expectations": [ + "Loads box-model.md and/or verse-development.md", + "Names the travelogue diagnosis", + "Proposes at least one division-of-labor formula (You-I-We / Past-Present-Future / other axis)", + "Mentions a 'what came before' diagnostic OR an equivalent reframing question" + ] + }, + { + "id": 2, + "name": "bridge-or-no-bridge", + "prompt": "/songwriting:song-form bridge I have a V/Ch/V/Ch and it feels long. Do I need a bridge or should I cut?", + "expected_output": "Routes to the bridge action. Loads bridge.md, templates/bridge-writing-prompt.md, form.md, and song-forms.md. Asks the missing-angle question (what hasn't the song said yet?), names the bridge functions, and — if no missing angle exists — recommends an alternative (e.g. distillation) instead of a bridge.", + "files": [], + "expectations": [ + "Loads bridge.md via the `bridge` action", + "Asks what the song hasn't said yet", + "Names at least 2 of the bridge's documented functions", + "If no missing angle is identified, recommends an alternative (distill, reshape form, etc.) instead of a bridge", + "Does NOT auto-prescribe a bridge without checking for a missing angle first" + ] + } + ] +} diff --git a/plugins/songwriting/skills/workflow/evals/evals.json b/plugins/songwriting/skills/workflow/evals/evals.json new file mode 100644 index 000000000..0b18919ec --- /dev/null +++ b/plugins/songwriting/skills/workflow/evals/evals.json @@ -0,0 +1,45 @@ +{ + "skill_name": "workflow", + "evals": [ + { + "id": 1, + "name": "brainstorm-blank-page", + "prompt": "/songwriting:workflow brainstorm I want to write a song but I have no idea where to start. Just a blank page.", + "expected_output": "Routes to the brainstorm action (Scenario 8). Loads context/pat-pattison/research/brainstorm.md and templates/brainstorm-opener.md. Presents 3-4 entry paths (sense-bound timed write / lived-input pull / recent envy / title cascade) without pushing for a title or form decision, and asks the writer to pick one path. Runs response-filter Coaching posture before emission.", + "files": [], + "expectations": [ + "Loads brainstorm.md and/or templates/brainstorm-opener.md via the `brainstorm` action", + "Presents at least 3 distinct entry paths", + "Does NOT ask for a title or form decision yet", + "Includes a timed-write option (e.g. a 10-minute dive) per Pat's discipline", + "Names the response-filter Coaching posture section's pass/fail/skip status before emission" + ] + }, + { + "id": 2, + "name": "idea-to-title-from-seed", + "prompt": "/songwriting:workflow idea \"the feeling of walking past your old apartment building five years later\"", + "expected_output": "Routes to the idea action. Loads idea-to-title.md, templates/idea-to-title-prompt.md, and object-writing.md. Walks through seed distillation (what / who / to whom / why now / underneath emotion) before generating titles. Proposes 10-15 title candidates with stressed-vowel and form-fit analysis, surfaces 2-3 finalists, and lets the writer choose rather than picking a winner.", + "files": [], + "expectations": [ + "Asks distillation questions (what/who/to whom/why now/underlying emotion) before generating titles", + "Generates 10-15 title candidates", + "Per candidate: notes stressed vowel, stress count, and form fit", + "Does NOT recommend a single title — surfaces finalists and lets the writer choose" + ] + }, + { + "id": 3, + "name": "fragment-development", + "prompt": "/songwriting:workflow fragment \"she keeps the porch light burning\"", + "expected_output": "Routes to the fragment action. Loads fragment-development.md, templates/fragment-development-prompt.md, and verse-development.md. Inventories the fragment (shape, likely position, what's strong/missing), surfaces implications (speaker, audience, time, place, tone), and recommends object-writing the implied world before drafting outward.", + "files": [], + "expectations": [ + "Inventories the fragment (shape, position-tendency, what's strong, what's missing)", + "Surfaces implications (speaker, audience, time, place, tone)", + "Recommends a timed object-write of the implied world BEFORE drafting", + "Does NOT immediately draft verses around the fragment" + ] + } + ] +} diff --git a/plugins/source-control/.claude-plugin/plugin.json b/plugins/source-control/.claude-plugin/plugin.json index 850ae0a70..dc3073c25 100644 --- a/plugins/source-control/.claude-plugin/plugin.json +++ b/plugins/source-control/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "source-control", - "version": "0.1.1", + "version": "0.2.0", "description": "Git and GitHub delivery workflow: /commit (Conventional Commits + Co-Authored-By trailer via safe heredoc mechanics), /pull-request (prep, create, CI monitoring, review-comment triage, merge, multi-PR babysit loop), and /worktree (create, status, cleanup, audit for parallel-session isolation).", "author": { "name": "Melodic Software", diff --git a/plugins/source-control/CHANGELOG.md b/plugins/source-control/CHANGELOG.md new file mode 100644 index 000000000..310b483af --- /dev/null +++ b/plugins/source-control/CHANGELOG.md @@ -0,0 +1,10 @@ +# Changelog + +All notable changes to the `source-control` plugin are documented here. Format follows +[Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. + +## [0.2.0] + +### Added + +- Readiness security-gate, mixed-actor, and three worktree evals. diff --git a/plugins/source-control/skills/pull-request/evals/evals.json b/plugins/source-control/skills/pull-request/evals/evals.json index 05943d4ec..896608067 100644 --- a/plugins/source-control/skills/pull-request/evals/evals.json +++ b/plugins/source-control/skills/pull-request/evals/evals.json @@ -61,6 +61,32 @@ "A code fix is gated on researched multi-source consensus about the cause, not a hunch", "It respects the max 3 CI fix iterations guard rather than looping fix-push-fail indefinitely" ] + }, + { + "id": 6, + "name": "readiness-blocks-on-security-failure", + "prompt": "Is PR #161 ready to merge? Its live gh state is captured in evals/fixtures/ci-status-security-fail.md relative to the skill directory.", + "expected_output": "Reports NOT ready. Per readiness.md Gate 3, the security check-run FAILURE is surfaced as a blocking finding requiring triage before merge, every non-pass check is classified, and the absence of reviewer comments is treated as 'cooldown still pending' (Gate 5), not 'ready'.", + "files": ["evals/fixtures/ci-status-security-fail.md"], + "expectations": [ + "Reports the PR as NOT ready to merge", + "Classifies the security check-run FAILURE explicitly and requires security triage before any merge suggestion, even if it may be a test-fixture false positive", + "Treats zero reviewer comments as cooldown-pending (Gate 5), not as evidence of readiness", + "Does not suggest or perform a merge while an unclassified FAILURE check exists (Gate 1/Gate 2/Gate 3 all open)" + ] + }, + { + "id": 7, + "name": "mixed-actor-bot-fixed-human-paused-same-turn", + "prompt": "Address the open review comments on PR #150; the branch is checked out. A bot reviewer left a comment flagging a missing null check on line 42. A human reviewer separately left a comment asking to rename a function for clarity. Handle both.", + "expected_output": "In the same turn, the bot finding is verified against the actual code, confirmed real, and fixed autonomously (react + reply + scoped fix) without waiting for user approval. The human reviewer's comment, evaluated in the same turn, pauses for explicit user approval before any reaction, reply, or fix — and the human-authored thread is never auto-resolved.", + "files": [], + "expectations": [ + "Distinguishes the bot actor from the human actor and applies different handling rules to each, within the same turn", + "Verifies the bot's finding against the actual code, then reacts, replies, and applies the fix autonomously once confirmed VALID — no user-approval pause for the bot finding", + "Pauses for explicit user approval before reacting to, replying to, or acting on the human reviewer's comment, even though the bot finding in the same turn was handled autonomously", + "Does not auto-resolve the human-authored review thread" + ] } ] } diff --git a/plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md b/plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md new file mode 100644 index 000000000..c663058ec --- /dev/null +++ b/plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md @@ -0,0 +1,44 @@ +# Fixture: CI + comment state on PR #161 (readiness check) + +The user is asking whether this PR is ready to merge. Most checks are green, but +one check is a FAILURE from a security check-run and no reviewer has commented yet. +A naive read of "mostly green" would wrongly declare ready. + +## `gh pr view 161 --json state,mergeable` + +```json +{ "state": "OPEN", "mergeable": "MERGEABLE" } +``` + +## `gh pr checks 161` + +``` +build-test-format pass 2m40s +markdown-lint pass 28s +shell-lint pass 44s +security-check fail 19s +static-analysis pass 3m11s +``` + +## Comments + +- Review-thread comments: (none) +- Issue-level comments: (none) +- PR reviews: (none) + +## `fetch-logs 161` (security-check annotations) + +``` +::error::security-check detected a potential secret at +src/config/tests/fixtures/example-settings.test.json:14 + rule: generic-api-key +``` + +## Ground truth (for grader — NOT visible to the skill under test) + +- The security-check FAILURE is a security finding → MUST be triaged and classified + (real leak vs test-fixture false positive) before merge is even suggested (Gate 3). +- Zero reviewer comments does NOT mean "ready" — the cooldown gate (Gate 5) and + reviewer window still apply. +- Correct verdict: NOT ready to merge; report the FAILURE as a blocker requiring + security triage; do not suggest merge. diff --git a/plugins/source-control/skills/worktree/evals/evals.json b/plugins/source-control/skills/worktree/evals/evals.json index 8ffbc3b1e..5fb1f0fb3 100644 --- a/plugins/source-control/skills/worktree/evals/evals.json +++ b/plugins/source-control/skills/worktree/evals/evals.json @@ -59,6 +59,45 @@ "The skill detects it is on the default branch and guides toward creating a branch or worktree before writing code", "It suggests the project's branch-naming convention (default /) as guidance, not a hard-enforced gate" ] + }, + { + "id": 6, + "name": "cleanup-dry-run-reports-only", + "prompt": "/worktree cleanup --dry-run", + "expected_output": "Identifies cleanup candidates across every worktree root the project uses (including a bare-clone hub, when detected) and reports them in a table without removing anything or running any destructive git command — no prune-then-remove, no branch deletion.", + "files": [], + "expectations": [ + "Scans every worktree root the project uses for orphan directories and prunable/stale/merged candidates", + "Identifies each candidate by reason (orphaned directory / prunable / PR merged / stale)", + "Extracts the real branch name from `git worktree list --porcelain` output, not from the directory name", + "Takes NO action — reports candidates only and exits, running no removal or branch-deletion command" + ] + }, + { + "id": 7, + "name": "create-invalid-name-rejected", + "prompt": "/worktree create my bad name!", + "expected_output": "Detects the invalid characters (the spaces and the `!`) against the EnterWorktree name constraints, explains what is wrong, and asks for a corrected name — it does NOT call EnterWorktree with the invalid name.", + "files": [], + "expectations": [ + "Identifies the space and `!` characters as invalid per the segment charset (letters, digits, dots, underscores, dashes only)", + "Explains the constraint being violated and asks the user for a corrected name", + "Does NOT call EnterWorktree with the invalid name — no worktree or branch is created" + ] + }, + { + "id": 8, + "name": "status-batched-gh-and-graceful-degrade", + "prompt": "/worktree status", + "expected_output": "Parses `git worktree list --porcelain` (CR-stripped), cross-references open PRs via a single batched `gh pr list` call (not one call per worktree), computes staleness against the WORKTREE_STALE_DAYS threshold (default 14), classifies each worktree, and presents a per-worktree table — degrading gracefully (skipping PR info with a note, not aborting) if `gh` is unavailable or fails.", + "files": [], + "expectations": [ + "Runs `git worktree list --porcelain` and strips carriage returns on Windows/Git Bash", + "Runs `gh pr list` ONCE as a batch cross-reference, not once per worktree branch", + "Uses the WORKTREE_STALE_DAYS env var for the stale threshold, falling back to 14 when unset or invalid", + "Classifies each worktree (active / stale / in-review / merged / prunable / locked) in the presented table", + "Degrades gracefully when `gh` fails — skips PR info with a note instead of aborting the whole status report" + ] } ] } From 8b945f4a9f6473d6674e9212c2a4b0c551723702 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Wed, 15 Jul 2026 03:58:18 -0400 Subject: [PATCH 2/7] =?UTF-8?q?fix:=20address=20Codex=20review=20=E2=80=94?= =?UTF-8?q?=20fixture=20answer-key=20leakage,=20reply-gating=20contradicti?= =?UTF-8?q?on,=20offline-export=20branch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - source-control: grader ground-truth removed from the security fixture (already covered by eval 6 expectations); eval 7 no longer gates the required human classification reply — only reaction/fix pause - event-storming: --discover-bcs protocol documents the supplied-export input path (no Miro) alongside the live board-URL read; board-write step marked live-path-only Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011xHkkNc7CR98L8Xz9Mu7ZA --- plugins/event-storming/skills/simulation/SKILL.md | 6 +++--- .../skills/pull-request/evals/evals.json | 10 ++++++---- .../evals/fixtures/ci-status-security-fail.md | 9 --------- 3 files changed, 9 insertions(+), 16 deletions(-) diff --git a/plugins/event-storming/skills/simulation/SKILL.md b/plugins/event-storming/skills/simulation/SKILL.md index 8e393ec04..cdd2d16c5 100644 --- a/plugins/event-storming/skills/simulation/SKILL.md +++ b/plugins/event-storming/skills/simulation/SKILL.md @@ -112,19 +112,19 @@ When invoked with `--evaluate`, run the iteration workflow against existing boar ## Bounded Context Discovery Protocol (`--discover-bcs`) -When invoked with `--discover-bcs [board-url]`, run Brandolini's 6 heuristics (Ch. 6) against an existing Big Picture board. This is the architect's post-workshop homework — reproducible and evidence-based. +When invoked with `--discover-bcs [board-url]`, run Brandolini's 6 heuristics (Ch. 6) against an existing Big Picture board. This is the architect's post-workshop homework — reproducible and evidence-based. The board data can arrive two ways: a board URL (live Miro read — requires Miro per the availability gate) or a directly-supplied board export (a structured-markdown board dump), which substitutes for the live read and needs no Miro. **Prerequisites:** A completed Big Picture board with People & Systems and Walk-through phases done. The more phases completed, the richer the signals. **Execution sequence:** -1. **Read ALL board items** via `miro_list_board_items` (full pagination). Parse into structured data: events by persona, people, external systems, hot spots, pivotal events. +1. **Read ALL board items** — via `miro_list_board_items` (full pagination) for a live board, or by parsing the supplied board export when one was provided instead of a URL. Parse into structured data: events by persona, people, external systems, hot spots, pivotal events. 2. **Apply Brandolini's 6 boundary heuristics mechanically** against the parsed data — canonical definitions in `/event-storming:methodology --big-picture` "Heuristics for Discovering Boundaries". Board-data signals: pivotal-event stickies (`dark_blue` / `--- PIVOTAL ---`) mark business-phase boundaries (H1); persona y-offset rows reveal parallel swimlanes (H2); per-persona event density per timeline zone assigns ownership (H3/H4); `[DIVERGENCE]` / hot-spot markers and same-noun-different-meaning phrasings signal boundaries (H5/H6). Use short BC names (2-3 words). 3. **Produce the BC analysis output** — a table of `# | BC Name (2-3 words) | Key Events | Primary Personas | Heuristic Evidence`, where the evidence column cites which heuristic fired (e.g. `H1: phase X→Y; H5: "Budget Approved" divergence; H6: "Ticket" means different things`). -4. **Place BC labels** on the board as cyan stickies at y=7100 (the canonical BC Labels row — bottom of board, below all other content per the Big Picture Y-Coordinate Table in `@./reference/miro-integration.md`), with `[BC]` prefix. +4. **Place BC labels** on the board as cyan stickies at y=7100 (the canonical BC Labels row — bottom of board, below all other content per the Big Picture Y-Coordinate Table in `@./reference/miro-integration.md`), with `[BC]` prefix. Live-board path only — when working from a supplied export there is no board to write; the step 3 table is the complete deliverable. 5. **Cross-reference with arrow voting winner** — which BC does the winner scope to? Mark it as the recommended next exploration target. diff --git a/plugins/source-control/skills/pull-request/evals/evals.json b/plugins/source-control/skills/pull-request/evals/evals.json index 896608067..f4b9ea530 100644 --- a/plugins/source-control/skills/pull-request/evals/evals.json +++ b/plugins/source-control/skills/pull-request/evals/evals.json @@ -67,7 +67,9 @@ "name": "readiness-blocks-on-security-failure", "prompt": "Is PR #161 ready to merge? Its live gh state is captured in evals/fixtures/ci-status-security-fail.md relative to the skill directory.", "expected_output": "Reports NOT ready. Per readiness.md Gate 3, the security check-run FAILURE is surfaced as a blocking finding requiring triage before merge, every non-pass check is classified, and the absence of reviewer comments is treated as 'cooldown still pending' (Gate 5), not 'ready'.", - "files": ["evals/fixtures/ci-status-security-fail.md"], + "files": [ + "evals/fixtures/ci-status-security-fail.md" + ], "expectations": [ "Reports the PR as NOT ready to merge", "Classifies the security check-run FAILURE explicitly and requires security triage before any merge suggestion, even if it may be a test-fixture false positive", @@ -79,14 +81,14 @@ "id": 7, "name": "mixed-actor-bot-fixed-human-paused-same-turn", "prompt": "Address the open review comments on PR #150; the branch is checked out. A bot reviewer left a comment flagging a missing null check on line 42. A human reviewer separately left a comment asking to rename a function for clarity. Handle both.", - "expected_output": "In the same turn, the bot finding is verified against the actual code, confirmed real, and fixed autonomously (react + reply + scoped fix) without waiting for user approval. The human reviewer's comment, evaluated in the same turn, pauses for explicit user approval before any reaction, reply, or fix — and the human-authored thread is never auto-resolved.", + "expected_output": "In the same turn, the bot finding is verified against the actual code, confirmed real, and fixed autonomously (react + reply + scoped fix) without waiting for user approval. The human reviewer's comment, evaluated in the same turn, still receives an evidence-backed classification reply (replies are required and not gated), but the skill pauses for explicit user approval before any reaction or fix for the human comment — and the human-authored thread is never auto-resolved.", "files": [], "expectations": [ "Distinguishes the bot actor from the human actor and applies different handling rules to each, within the same turn", "Verifies the bot's finding against the actual code, then reacts, replies, and applies the fix autonomously once confirmed VALID — no user-approval pause for the bot finding", - "Pauses for explicit user approval before reacting to, replying to, or acting on the human reviewer's comment, even though the bot finding in the same turn was handled autonomously", + "Posts an evidence-backed classification reply to the human reviewer's comment (replies are not gated), but pauses for explicit user approval before reacting to or acting on it, even though the bot finding in the same turn was handled autonomously", "Does not auto-resolve the human-authored review thread" ] } ] -} +} \ No newline at end of file diff --git a/plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md b/plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md index c663058ec..0e9494ab4 100644 --- a/plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md +++ b/plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md @@ -33,12 +33,3 @@ static-analysis pass 3m11s src/config/tests/fixtures/example-settings.test.json:14 rule: generic-api-key ``` - -## Ground truth (for grader — NOT visible to the skill under test) - -- The security-check FAILURE is a security finding → MUST be triaged and classified - (real leak vs test-fixture false positive) before merge is even suggested (Gate 3). -- Zero reviewer comments does NOT mean "ready" — the cooldown gate (Gate 5) and - reviewer window still apply. -- Correct verdict: NOT ready to merge; report the FAILURE as a blocker requiring - security triage; do not suggest merge. From 0f5f761a69871c74d28a412cface88d4d384c643 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Wed, 15 Jul 2026 04:27:27 -0400 Subject: [PATCH 3/7] =?UTF-8?q?fix:=20address=20Codex=20review=20=E2=80=94?= =?UTF-8?q?=20eval=20reproducibility=20and=20hygiene?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - review-toolkit: evals 7-20 files arrays held scenario prose, not fixture paths; prose folded into prompts, arrays emptied - discovery: comparison eval targets real tools (RabbitMQ/Kafka/NATS) so primary-source expectations are satisfiable - source-control: evals.json rewritten with LF + final newline (editorconfig hygiene failure from the previous rewrite) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011xHkkNc7CR98L8Xz9Mu7ZA --- .../skills/research/evals/evals.json | 8 +-- .../code-review-fanout/evals/evals.json | 56 +++++++++---------- .../skills/pull-request/evals/evals.json | 2 +- 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/plugins/discovery/skills/research/evals/evals.json b/plugins/discovery/skills/research/evals/evals.json index 7e9c94484..61018ff5d 100644 --- a/plugins/discovery/skills/research/evals/evals.json +++ b/plugins/discovery/skills/research/evals/evals.json @@ -97,17 +97,17 @@ { "id": 8, "name": "broad-topic-triple-tool-comparison", - "prompt": "We're choosing among three competing message-queue libraries for a new service — Queue Tool A, Queue Tool B, and Queue Tool C. Research which is the better fit for our throughput and durability requirements.", - "expected_output": "Broad-topic auto-detect fires — 3 tools named in an explicit X vs Y vs Z comparison — so doubled minimums apply: 6+ queries per phase, 12+ total, 5+ distinct tool types across the topic, 4+ Tier 0/1 sources per claim. A written Phase 1 gap/conflict/hypothesis block drives Phase 2, which includes the mandatory falsification query. Phase 3 uses the tool-ecosystem fallback (Path B) since no preferred-source author covers these three placeholder tools. Outcome gate runs before presenting.", + "prompt": "We're choosing among three competing message-queue libraries for a new service — RabbitMQ, Apache Kafka, and NATS. Research which is the better fit for our throughput and durability requirements.", + "expected_output": "Broad-topic auto-detect fires — 3 tools named in an explicit X vs Y vs Z comparison — so doubled minimums apply: 6+ queries per phase, 12+ total, 5+ distinct tool types across the topic, 4+ Tier 0/1 sources per claim. A written Phase 1 gap/conflict/hypothesis block drives Phase 2, which includes the mandatory falsification query. Phase 3 uses the tool-ecosystem fallback (Path B) since no preferred-source author covers these three tools (no consumer preferred-source roster exists in a bare checkout). Outcome gate runs before presenting.", "files": [], "expectations": [ "Detects the broad-topic trigger (3 distinct tools named, explicit comparison) and applies DOUBLED minimums — 6+ Phase 1 queries, 12+ total queries across all phases, 5+ distinct tool types, 4+ Tier 0/1 sources per accepted claim", "Phase 1 launches 6+ queries (not the standard 3+) using 3+ distinct tool types", "Emits a WRITTEN Phase 1 analysis block — leading hypothesis + numbered gaps + numbered conflicts + tool-diversity audit + recency status + falsification candidate — before composing any Phase 2 query", "Phase 2 query count scales to the numbered gaps/conflicts from Phase 1, not a flat 6; each query traces to a named entry in the written block", - "Phase 2 includes exactly one mandatory falsification query attempting to break the Phase 1 leading hypothesis (e.g., 'Queue Tool A durability guarantees disputed' or 'Queue Tool B does NOT support clustering')", + "Phase 2 includes exactly one mandatory falsification query attempting to break the Phase 1 leading hypothesis (e.g., 'RabbitMQ durability guarantees disputed' or 'NATS does NOT support clustering')", "Phase 2 fetches each of the three tools' primary docs/changelog directly this turn rather than accepting a SERP or synthesis summary", - "Phase 3 fires the tool-ecosystem fallback (Path B) because no preferred-source roster names an author covering these three placeholder tools — cites official maintainer source, upstream changelog/releases, and one recognized industry authority for each", + "Phase 3 fires the tool-ecosystem fallback (Path B) because no preferred-source roster names an author covering these three tools (no consumer preferred-source roster exists in a bare checkout) — cites official maintainer source, upstream changelog/releases, and one recognized industry authority for each", "Evidence table tracks Tier + tool-diversity + confidence per claim; rejects any claim whose sources are all Tier 2 or all from one tool category", "Source-tier ratio enforced — every accepted claim has >=1 Tier 0/1 source plus >=2 independent corroborators; three citations of the same secondary pool count as one source, not three", "Confidence calibration applied — every comparison claim marked HIGH/MEDIUM/LOW; LOW-confidence claims are not used as the basis for a recommendation", diff --git a/plugins/review-toolkit/skills/code-review-fanout/evals/evals.json b/plugins/review-toolkit/skills/code-review-fanout/evals/evals.json index dc5e94ce0..55669d093 100644 --- a/plugins/review-toolkit/skills/code-review-fanout/evals/evals.json +++ b/plugins/review-toolkit/skills/code-review-fanout/evals/evals.json @@ -76,9 +76,9 @@ { "id": 7, "name": "tier-small-leaf-agents-only", - "prompt": "Fan out a review of this small 30-line change — nothing security-sensitive, no layer boundaries crossed.", + "prompt": "[Scenario: a 30-line non-security, non-layer-boundary change staged.] Fan out a review of this small 30-line change — nothing security-sensitive, no layer boundaries crossed.", "expected_output": "The small tier dispatches code-reviewer only (plus security-reviewer solely when a security-sensitive path is touched) and does not promote to medium or large.", - "files": ["a 30-line non-security, non-layer-boundary change staged"], + "files": [], "expectations": [ "Classifies the change as tier small (<50 changed lines) and does not promote to medium", "Dispatches code-reviewer only; does NOT dispatch architecture-guardian or any orchestrator plugin at this tier", @@ -88,9 +88,9 @@ { "id": 8, "name": "tier-small-security-promotes-to-medium", - "prompt": "Fan out a review of this 25-line change — it touches an authentication check.", + "prompt": "[Scenario: a 25-line change touching an auth/identity path.] Fan out a review of this 25-line change — it touches an authentication check.", "expected_output": "A small-sized diff that is security-sensitive promotes to the full medium-tier surface set rather than only adding security-reviewer to the small set.", - "files": ["a 25-line change touching an auth/identity path"], + "files": [], "expectations": [ "Detects the security-sensitive path and promotes the diff from small to medium tier despite the line count", "Dispatches the full medium-tier surface set (code-reviewer, security-reviewer, orchestrator plugin(s), and architecture-guardian when structural paths are touched) rather than only adding security-reviewer to the small set", @@ -100,9 +100,9 @@ { "id": 9, "name": "large-tier-ownerless-slices-and-exclusions", - "prompt": "Fan out a review of this 400-line change spanning multiple directories and ecosystems.", + "prompt": "[Scenario: a >300-line cross-cutting change across multiple directories.] Fan out a review of this 400-line change spanning multiple directories and ecosystems.", "expected_output": "The large tier dispatches the medium set plus the project's ownerless review-criteria docs as slice-subagents, while excluding agent-owned criteria and the two non-diff-review agents.", - "files": ["a >300-line cross-cutting change across multiple directories"], + "files": [], "expectations": [ "Classifies tier as large (>300 lines or cross-cutting) and dispatches the medium set plus discovered ownerless review-criteria slices", "Does NOT dispatch code-quality/security/architecture criteria docs as separate slice-subagents — those are agent-owned, so re-reviewing them would be pure waste", @@ -113,9 +113,9 @@ { "id": 10, "name": "dedup-cross-surface-max-severity", - "prompt": "Fan out a review where code-reviewer and security-reviewer both flag the same file:line.", + "prompt": "[Scenario: a change where two surfaces both flag the same file:line.] Fan out a review where code-reviewer and security-reviewer both flag the same file:line.", "expected_output": "Stage 3 dedup merges the same-issue findings from two surfaces into one ranked finding keyed by file plus line-proximity, carrying MAX severity and MAX confidence and listing both surfaces.", - "files": ["a change where two surfaces both flag the same file:line"], + "files": [], "expectations": [ "Merges the two same-issue findings into ONE record keyed by normalized file path + line-proximity bucket (±3 lines), not by category", "Merged severity is the MAX of the contributing surfaces' severities, never a filtered-down value", @@ -125,9 +125,9 @@ { "id": 11, "name": "plugin-findings-severity-derived-not-invented", - "prompt": "Fan out a review where the code-review orchestrator plugin returns confidence-filtered issues with no native severity.", + "prompt": "[Scenario: a PR-present change where the code-review plugin emits confidence-filtered issues.] Fan out a review where the code-review orchestrator plugin returns confidence-filtered issues with no native severity.", "expected_output": "The severity crosswalk derives a tier from finding content for surfaces with no native severity; a high confidence score is never treated as a severity tier.", - "files": ["a PR-present change where the code-review plugin emits confidence-filtered issues"], + "files": [], "expectations": [ "Findings from a surface with no native severity get a tier DERIVED from content (bug/correctness vs. convention-adherence), not copied from its confidence score", "A high (>=80) confidence-filtered score maps to the confidence enum 'high', NOT to a high severity tier", @@ -137,9 +137,9 @@ { "id": 12, "name": "findings-file-stable-shape-relative-paths", - "prompt": "Fan out a review and persist the findings.", + "prompt": "[Scenario: any reviewable change.] Fan out a review and persist the findings.", "expected_output": "The persisted findings file has the stable frontmatter-plus-table-plus-appendix shape and cites file:line only as repo-relative paths.", - "files": ["any reviewable change"], + "files": [], "expectations": [ "Findings file frontmatter declares type: review-findings, plus date, branch, and tier", "Body includes the ranked Findings table, a ## Unparsed appendix, and a ## Surfaces reconciliation line", @@ -149,9 +149,9 @@ { "id": 13, "name": "clean-tree-no-pr-spawns-nothing", - "prompt": "Fan out a review — my branch is clean and has no open PR.", + "prompt": "[Scenario: empty git diff vs. merge-base, branch not ahead of base, no open PR.] Fan out a review — my branch is clean and has no open PR.", "expected_output": "A truly clean tree (no diff, not ahead of base, no open PR) reports that there is nothing to review and spawns nothing.", - "files": ["empty git diff vs. merge-base, branch not ahead of base, no open PR"], + "files": [], "expectations": [ "Reports that there are no changes to review", "Spawns no reviewer surfaces", @@ -161,9 +161,9 @@ { "id": 14, "name": "orchestrator-absent-degrades-gracefully", - "prompt": "Fan out a medium-tier review — none of the optional orchestrator plugins are installed.", + "prompt": "[Scenario: a medium-tier change with no orchestrator plugin installed.] Fan out a medium-tier review — none of the optional orchestrator plugins are installed.", "expected_output": "An absent orchestrator plugin degrades to leaf-only coverage with an explicit skip notice; normalization still runs over whatever surfaces produced output.", - "files": ["a medium-tier change with no orchestrator plugin installed"], + "files": [], "expectations": [ "Dispatches only this plugin's leaf agents (and slice-subagents where applicable) when an orchestrator plugin is absent", "Emits an explicit notice naming that orchestrator breadth was skipped rather than silently omitting it", @@ -173,9 +173,9 @@ { "id": 15, "name": "run-everything-workflows-disabled-degraded-block", - "prompt": "/review-toolkit:code-review-fanout run-everything", + "prompt": "[Scenario: CLAUDE_CODE_DISABLE_WORKFLOWS=1 set in the environment; a reviewable change.] /review-toolkit:code-review-fanout run-everything", "expected_output": "The pre-launch gate detects Workflows are disabled and falls back to the main-thread coverage-parity roster with a DEGRADED block prepended, rather than attempting a launch.", - "files": ["CLAUDE_CODE_DISABLE_WORKFLOWS=1 set in the environment; a reviewable change"], + "files": [], "expectations": [ "The pre-launch availability gate detects the disabled state BEFORE attempting any Workflow launch", "Falls back to spawning the same leaf roster on the main thread — findings coverage stays full", @@ -185,9 +185,9 @@ { "id": 16, "name": "run-everything-null-leaf-named-not-dropped", - "prompt": "/review-toolkit:code-review-fanout run-everything", + "prompt": "[Scenario: a run where one leaf returns null.] /review-toolkit:code-review-fanout run-everything", "expected_output": "A leaf that returns null (budget exhaustion, a Bash failure, etc.) is named under a 'Returned no result' line, never silently absorbed into the roster count.", - "files": ["a run where one leaf returns null"], + "files": [], "expectations": [ "Reconciles the expected roster (ran) against the non-null returns rather than reporting only the non-null count", "Names the null leaf explicitly under the ## Surfaces 'Returned no result' line, with cause when known", @@ -197,9 +197,9 @@ { "id": 17, "name": "run-everything-priority-ordering-tier1-barrier", - "prompt": "/review-toolkit:code-review-fanout run-everything", + "prompt": "[Scenario: a run under a finite user turn-token budget that exhausts partway through the leaf fan-out.] /review-toolkit:code-review-fanout run-everything", "expected_output": "Tier-1 high-value reviewers (security-reviewer, architecture-guardian, code-reviewer) run as a barrier before tier-2 leaves, so budget exhaustion drops tier-2, never tier-1.", - "files": ["a run under a finite user turn-token budget that exhausts partway through the leaf fan-out"], + "files": [], "expectations": [ "security-reviewer, architecture-guardian, and code-reviewer run as tier 1, awaited as a barrier before any tier-2 leaf starts", "When the budget exhausts during tier 2, those calls resolve to null rather than rejecting or aborting the whole run", @@ -209,9 +209,9 @@ { "id": 18, "name": "fix-pass-splits-mixed-class-findings", - "prompt": "/review-toolkit:code-review-fanout fix", + "prompt": "[Scenario: a current-branch findings file with both a cleanup-class (extract-helper) finding and a correctness-class (CRITICAL security) finding.] /review-toolkit:code-review-fanout fix", "expected_output": "Fix-pass classifies findings by class and routes cleanup-class findings to /simplify while applying correctness-class findings via sequential scope-fenced fixes.", - "files": ["a current-branch findings file with both a cleanup-class (extract-helper) finding and a correctness-class (CRITICAL security) finding"], + "files": [], "expectations": [ "Locates the newest findings file for the CURRENT branch only", "Classifies the cleanup-class finding and routes it to the bundled /simplify skill", @@ -222,9 +222,9 @@ { "id": 19, "name": "fix-pass-no-findings-branch-scoped", - "prompt": "/review-toolkit:code-review-fanout fix", + "prompt": "[Scenario: the current branch has no findings file; a different branch's findings directory does have one.] /review-toolkit:code-review-fanout fix", "expected_output": "Fix-pass with no findings file for the current branch reports cleanly and stops, without scanning another branch's findings.", - "files": ["the current branch has no findings file; a different branch's findings directory does have one"], + "files": [], "expectations": [ "Reports 'No findings for branch ' and stops", "Does NOT scan, read, or apply another branch's findings file — the branch check against frontmatter is the safety fence", @@ -234,9 +234,9 @@ { "id": 20, "name": "fix-pass-correctness-never-simplify", - "prompt": "/review-toolkit:code-review-fanout fix", + "prompt": "[Scenario: a findings file whose only entries are correctness-class (a logic bug and a race condition).] /review-toolkit:code-review-fanout fix", "expected_output": "All correctness findings are applied via sequential scope-fenced fixes; /simplify is never invoked for a correctness-class finding.", - "files": ["a findings file whose only entries are correctness-class (a logic bug and a race condition)"], + "files": [], "expectations": [ "Applies both correctness findings via sequential, non-parallel scope-fenced fixes — one file at a time", "Never invokes /simplify for a correctness-class finding — /simplify is cleanup-only and would silently drop the bug", diff --git a/plugins/source-control/skills/pull-request/evals/evals.json b/plugins/source-control/skills/pull-request/evals/evals.json index f4b9ea530..eda257951 100644 --- a/plugins/source-control/skills/pull-request/evals/evals.json +++ b/plugins/source-control/skills/pull-request/evals/evals.json @@ -91,4 +91,4 @@ ] } ] -} \ No newline at end of file +} From 0a333dd9fe0f1f17e16ebdbb18e16f1b02bb6382 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Wed, 15 Jul 2026 04:48:53 -0400 Subject: [PATCH 4/7] =?UTF-8?q?fix:=20address=20Codex=20review=20=E2=80=94?= =?UTF-8?q?=20name=20research=20target,=20mutation-free=20dry-run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - discovery: production-readiness eval names FusionCache so primary-source expectations are satisfiable - source-control: cleanup --dry-run now specifies git worktree prune --dry-run (report-only) in Step 1; eval 6 wording aligned. A dry-run that mutates worktree metadata violated its own contract Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011xHkkNc7CR98L8Xz9Mu7ZA --- plugins/discovery/skills/research/evals/evals.json | 4 ++-- plugins/source-control/skills/worktree/context/cleanup.md | 2 ++ plugins/source-control/skills/worktree/evals/evals.json | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/plugins/discovery/skills/research/evals/evals.json b/plugins/discovery/skills/research/evals/evals.json index 61018ff5d..10a5f1aa7 100644 --- a/plugins/discovery/skills/research/evals/evals.json +++ b/plugins/discovery/skills/research/evals/evals.json @@ -77,8 +77,8 @@ { "id": 7, "name": "floor-scaling-single-product", - "prompt": "We're deciding whether to adopt a particular open-source in-process cache library for a new service. Before committing, research its actual production readiness: current stability/release maturity, adapter support across the data stores we use, history of breaking changes, integration story with our dependency-injection container, testing/mocking support, and any documented limitations.", - "expected_output": "Single-product topic — broad-topic auto-detect does not fire (one library named, no comparison), so standard floors apply. The written Phase 1 gap list surfaces well more than 3 open questions (release/stability status, adapter maturity, breaking-change history, DI integration, testing support, known limitations), and Phase 2 runs one query per gap rather than stopping at the 3-query floor. Includes the mandatory falsification query, primaries fetched directly, and the outcome gate before presenting.", + "prompt": "We're deciding whether to adopt FusionCache (the open-source .NET in-process/hybrid cache library) for a new service. Before committing, research its actual production readiness: current stability/release maturity, adapter support across the data stores we use, history of breaking changes, integration story with our dependency-injection container, testing/mocking support, and any documented limitations.", + "expected_output": "Single-product topic — broad-topic auto-detect does not fire (one named library (FusionCache), no comparison), so standard floors apply. The written Phase 1 gap list surfaces well more than 3 open questions (release/stability status, adapter maturity, breaking-change history, DI integration, testing support, known limitations), and Phase 2 runs one query per gap rather than stopping at the 3-query floor. Includes the mandatory falsification query, primaries fetched directly, and the outcome gate before presenting.", "files": [], "expectations": [ "Does not treat this as a broad-topic / comparison — only one library is named, so standard (non-doubled) minimums apply", diff --git a/plugins/source-control/skills/worktree/context/cleanup.md b/plugins/source-control/skills/worktree/context/cleanup.md index 457868b39..1e674d2cc 100644 --- a/plugins/source-control/skills/worktree/context/cleanup.md +++ b/plugins/source-control/skills/worktree/context/cleanup.md @@ -12,6 +12,8 @@ git worktree prune Cleans up worktree administrative records for directories that no longer exist on disk (e.g., manually deleted via `rm -rf`). +In `--dry-run` mode this step runs `git worktree prune --dry-run` instead — it reports what would be pruned without touching worktree metadata, keeping the whole dry-run pass mutation-free. + ## Step 2: Identify cleanup candidates Run `status` logic internally and identify candidates: diff --git a/plugins/source-control/skills/worktree/evals/evals.json b/plugins/source-control/skills/worktree/evals/evals.json index 5fb1f0fb3..3cc34d57c 100644 --- a/plugins/source-control/skills/worktree/evals/evals.json +++ b/plugins/source-control/skills/worktree/evals/evals.json @@ -64,13 +64,13 @@ "id": 6, "name": "cleanup-dry-run-reports-only", "prompt": "/worktree cleanup --dry-run", - "expected_output": "Identifies cleanup candidates across every worktree root the project uses (including a bare-clone hub, when detected) and reports them in a table without removing anything or running any destructive git command — no prune-then-remove, no branch deletion.", + "expected_output": "Identifies cleanup candidates across every worktree root the project uses (including a bare-clone hub, when detected) and reports them in a table without removing anything or running any mutating git command — metadata pruning runs only as `git worktree prune --dry-run` (report-only), no removal, no branch deletion.", "files": [], "expectations": [ "Scans every worktree root the project uses for orphan directories and prunable/stale/merged candidates", "Identifies each candidate by reason (orphaned directory / prunable / PR merged / stale)", "Extracts the real branch name from `git worktree list --porcelain` output, not from the directory name", - "Takes NO action — reports candidates only and exits, running no removal or branch-deletion command" + "Takes NO mutating action — reports candidates only and exits; any prune runs in --dry-run form, and no removal or branch-deletion command runs" ] }, { From fdd0ed998bf61ef5bad3addd84b71bd2b2f0c895 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Wed, 15 Jul 2026 04:56:25 -0400 Subject: [PATCH 5/7] fix: align readiness eval with Gate 1 terminal-state semantics Fixture checks are all terminal, so Gate 1 is satisfied; only Gates 2-3 block. Expectation narrowed accordingly. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011xHkkNc7CR98L8Xz9Mu7ZA --- plugins/source-control/skills/pull-request/evals/evals.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source-control/skills/pull-request/evals/evals.json b/plugins/source-control/skills/pull-request/evals/evals.json index eda257951..dc3f46e78 100644 --- a/plugins/source-control/skills/pull-request/evals/evals.json +++ b/plugins/source-control/skills/pull-request/evals/evals.json @@ -74,7 +74,7 @@ "Reports the PR as NOT ready to merge", "Classifies the security check-run FAILURE explicitly and requires security triage before any merge suggestion, even if it may be a test-fixture false positive", "Treats zero reviewer comments as cooldown-pending (Gate 5), not as evidence of readiness", - "Does not suggest or perform a merge while an unclassified FAILURE check exists (Gate 1/Gate 2/Gate 3 all open)" + "Does not suggest or perform a merge while an unclassified FAILURE check exists — Gate 1 (all checks terminal) is satisfied in this fixture, but Gate 2 (every FAILURE classified) and Gate 3 (security triage) remain open" ] }, { From 405d82ee97f95d89951bb569151399e8ab1beb3b Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Wed, 15 Jul 2026 05:08:21 -0400 Subject: [PATCH 6/7] fix: mark Gate 2 satisfied once the security failure is classified Eval 6 required classification while asserting Gate 2 open. NOT-ready now rests on Gates 3 and 5 only. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011xHkkNc7CR98L8Xz9Mu7ZA --- plugins/source-control/skills/pull-request/evals/evals.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/source-control/skills/pull-request/evals/evals.json b/plugins/source-control/skills/pull-request/evals/evals.json index dc3f46e78..62d537e22 100644 --- a/plugins/source-control/skills/pull-request/evals/evals.json +++ b/plugins/source-control/skills/pull-request/evals/evals.json @@ -74,7 +74,7 @@ "Reports the PR as NOT ready to merge", "Classifies the security check-run FAILURE explicitly and requires security triage before any merge suggestion, even if it may be a test-fixture false positive", "Treats zero reviewer comments as cooldown-pending (Gate 5), not as evidence of readiness", - "Does not suggest or perform a merge while an unclassified FAILURE check exists — Gate 1 (all checks terminal) is satisfied in this fixture, but Gate 2 (every FAILURE classified) and Gate 3 (security triage) remain open" + "Does not suggest or perform a merge — Gate 1 (all checks terminal) is satisfied in this fixture, and classifying the security FAILURE satisfies Gate 2, but the PR stays NOT ready on Gate 3 (a classified security finding must be resolved, not just labeled) and Gate 5 (cooldown)" ] }, { From c0910aac00b2252ea0462e559289340ed505b9af Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Wed, 15 Jul 2026 05:15:49 -0400 Subject: [PATCH 7/7] fix: strip remaining readiness answer cue from security fixture Fixture preamble narrated the expected verdict to the skill under test. Fixture is raw gh output only; the cue lives in expected_output now. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_011xHkkNc7CR98L8Xz9Mu7ZA --- plugins/source-control/skills/pull-request/evals/evals.json | 2 +- .../pull-request/evals/fixtures/ci-status-security-fail.md | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/source-control/skills/pull-request/evals/evals.json b/plugins/source-control/skills/pull-request/evals/evals.json index 62d537e22..4ab6ecce8 100644 --- a/plugins/source-control/skills/pull-request/evals/evals.json +++ b/plugins/source-control/skills/pull-request/evals/evals.json @@ -66,7 +66,7 @@ "id": 6, "name": "readiness-blocks-on-security-failure", "prompt": "Is PR #161 ready to merge? Its live gh state is captured in evals/fixtures/ci-status-security-fail.md relative to the skill directory.", - "expected_output": "Reports NOT ready. Per readiness.md Gate 3, the security check-run FAILURE is surfaced as a blocking finding requiring triage before merge, every non-pass check is classified, and the absence of reviewer comments is treated as 'cooldown still pending' (Gate 5), not 'ready'.", + "expected_output": "Reports NOT ready. Per readiness.md Gate 3, the security check-run FAILURE is surfaced as a blocking finding requiring triage before merge, every non-pass check is classified, and the absence of reviewer comments is treated as 'cooldown still pending' (Gate 5), not 'ready'. A naive 'mostly green' read must not produce a ready verdict.", "files": [ "evals/fixtures/ci-status-security-fail.md" ], diff --git a/plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md b/plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md index 0e9494ab4..a98ef6bb4 100644 --- a/plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md +++ b/plugins/source-control/skills/pull-request/evals/fixtures/ci-status-security-fail.md @@ -1,8 +1,6 @@ # Fixture: CI + comment state on PR #161 (readiness check) -The user is asking whether this PR is ready to merge. Most checks are green, but -one check is a FAILURE from a security check-run and no reviewer has commented yet. -A naive read of "mostly green" would wrongly declare ready. +Captured `gh` command output for PR #161. ## `gh pr view 161 --json state,mergeable`