From ef4cbde69ce5bd85e2a99fab44317c9d2672ca23 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 12 Jul 2026 21:20:27 -0400 Subject: [PATCH 1/4] chore(evals): author skill evals for implementation (5 skills) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add rich-form evals/evals.json for implement, implement-dispatch, build, lint, and setup, and bump the implementation plugin 0.2.0 -> 0.3.0 so marketplace consumers pick up the new shipped eval component. All five skills warrant evals (no skip verdicts): each carries a judgment-bearing behavioral contract that could silently regress — implement (feature/fix/refactor/config mode routing, divergence detection, scope-fence over-correction guard, never-declare-impossible), implement-dispatch (autonomous-vs-interactive orchestration routing, scope-fenced briefs, verify-against-direct-evidence, conservative non-interactive divergence + DEVIATIONS.md, fresh-context verifier), build and lint (ecosystem-detection routing, skip-not-FAIL missing-tool guardrail, consumer-config-wins precedence, build-vs-lint scope split), and setup (infer-then-interview, idempotent read-existing, write-scope guardrail, ambiguous-inference-asks). Cases are modeled on the bug-report rich-form exemplar and validated against plugins/skill-quality/reference/evals.schema.json. Refs melodic-software/medley#1449 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../implementation/.claude-plugin/plugin.json | 2 +- plugins/implementation/CHANGELOG.md | 12 +++ .../skills/build/evals/evals.json | 66 +++++++++++++++ .../implement-dispatch/evals/evals.json | 80 +++++++++++++++++++ .../skills/implement/evals/evals.json | 79 ++++++++++++++++++ .../skills/lint/evals/evals.json | 66 +++++++++++++++ .../skills/setup/evals/evals.json | 54 +++++++++++++ 7 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 plugins/implementation/skills/build/evals/evals.json create mode 100644 plugins/implementation/skills/implement-dispatch/evals/evals.json create mode 100644 plugins/implementation/skills/implement/evals/evals.json create mode 100644 plugins/implementation/skills/lint/evals/evals.json create mode 100644 plugins/implementation/skills/setup/evals/evals.json diff --git a/plugins/implementation/.claude-plugin/plugin.json b/plugins/implementation/.claude-plugin/plugin.json index d441b4c06..93bfce409 100644 --- a/plugins/implementation/.claude-plugin/plugin.json +++ b/plugins/implementation/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://json.schemastore.org/claude-code-plugin-manifest.json", "name": "implementation", - "version": "0.2.0", + "version": "0.3.0", "description": "Disciplined implementation stage: execute approved plans inline or via orchestrated workers, run polyglot build/test/lint verification, author and diagnose tests, drive live E2E checks, and prove changes achieved their intended outcome — with measurable-improvement claims verified against planning-time baselines.", "author": { "name": "Melodic Software", diff --git a/plugins/implementation/CHANGELOG.md b/plugins/implementation/CHANGELOG.md index 55d8d1060..c16afa947 100644 --- a/plugins/implementation/CHANGELOG.md +++ b/plugins/implementation/CHANGELOG.md @@ -3,6 +3,18 @@ All notable changes to the `implementation` plugin are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); this plugin uses semantic versioning. +## [0.3.0] + +### Added + +- **Rich-form evals for five skills.** `evals/evals.json` ships for `implement`, `implement-dispatch`, + `build`, `lint`, and `setup` — the skills' judgment-bearing contracts (mode/orchestration routing, + divergence and scope-fence guardrails, skip-not-FAIL and consumer-config-precedence behavior, and the + config-writer's interview/write-scope discipline) are now covered by objectively-verifiable cases, + modeled on the `bug-report` rich-form exemplar and validated against + `plugins/skill-quality/reference/evals.schema.json`. Evals are a shipped component, so this minor bump + is their delivery vehicle; no behavioral change to the skills themselves. + ## [0.2.0] ### Changed diff --git a/plugins/implementation/skills/build/evals/evals.json b/plugins/implementation/skills/build/evals/evals.json new file mode 100644 index 000000000..7a5a5d9cb --- /dev/null +++ b/plugins/implementation/skills/build/evals/evals.json @@ -0,0 +1,66 @@ +{ + "skill_name": "build", + "evals": [ + { + "id": 1, + "name": "auto-detects-affected-ecosystems-from-git-status", + "prompt": "/implementation:build — I just edited a couple of .cs files and a pyproject-managed .py module.", + "expected_output": "Classifies the changed files from git status against each covered ecosystem's globs, detects the dotnet and python ecosystems, resolves each one's command surface via the ladder, and runs build -> test -> lint per ecosystem — reporting a per-ecosystem results table. It does not run ecosystems with no changed files.", + "files": [], + "expectations": [ + "Detects the affected ecosystems (dotnet, python) by matching changed files against ecosystem globs, not by building the whole repo", + "Runs build -> test -> lint in order for each affected ecosystem", + "Emits a per-ecosystem results table using pass / FAIL / skip / — statuses", + "Does not run ecosystems that have no changed files" + ] + }, + { + "id": 2, + "name": "missing-tool-reports-skip-not-fail", + "prompt": "/implementation:build python — but `uv` is not installed on this machine.", + "expected_output": "Verifies the tool is on PATH before running; since uv is missing it reports the python ecosystem as `skip` with the ecosystem's install-hint — it does NOT report FAIL for a missing tool.", + "files": [], + "expectations": [ + "Reports the ecosystem as `skip` (with the install-hint) rather than `FAIL` when the required tool is absent from PATH", + "Distinguishes a missing-tool skip from an actual build/test/lint failure", + "Does not treat the absent tool as a red build" + ] + }, + { + "id": 3, + "name": "consumer-ecosystem-config-wins-over-bundled-default", + "prompt": "/implementation:build dotnet — the repo has a tracked `.claude/ecosystems/dotnet.yaml` with a project-specific test-cmd.", + "expected_output": "Resolves the dotnet command surface through the four-rung ladder with the consumer's tracked `.claude/ecosystems/dotnet.yaml` authoritative (rung 1) — the bundled portable default is only the rung-4 fallback. It runs the consumer's declared commands, not the bundled ones.", + "files": [], + "expectations": [ + "Resolves commands via the ladder with the consumer's `.claude/ecosystems/dotnet.yaml` taking precedence (rung 1)", + "Uses the bundled portable default only as the rung-4 fallback when no consumer file is present", + "A malformed consumer file degrades to inference with a warning rather than hard-stopping" + ] + }, + { + "id": 4, + "name": "build-scope-excludes-lint-only-surfaces", + "prompt": "/implementation:build all — the working tree changed a GitHub Actions workflow YAML and a plain README, among other files.", + "expected_output": "Runs only the ecosystems /build covers (dotnet, python, typescript, bash, powershell, markdown). It does NOT run the lint-only `yaml` or `cross-cutting` surfaces — those belong to /lint, and cross-cutting's `**` glob is never matched here.", + "files": [], + "expectations": [ + "Runs only the six /build-covered ecosystems (dotnet, python, typescript, bash, powershell, markdown)", + "Does NOT run the `yaml` surface for the workflow file — that is /lint-only", + "Does NOT run the `cross-cutting` surface (its `**` glob is never matched by /build); points the user to /lint for those" + ] + }, + { + "id": 5, + "name": "clean-tree-falls-back-to-branch-diff", + "prompt": "/implementation:build — every block was already committed on this feature branch, so the working tree is clean.", + "expected_output": "With a clean working tree it falls back to the branch diff (git diff against the merge-base with the default branch) so checkpoint-committed work still gets classified and verified — rather than reporting 'no changes' and exiting.", + "files": [], + "expectations": [ + "Falls back to the branch diff vs the default-branch merge-base when the working tree is clean", + "Classifies and verifies the checkpoint-committed changes rather than reporting nothing to do", + "Only reports 'No changes found' when neither the working tree nor the branch diff yields changes and no ecosystem argument was given" + ] + } + ] +} diff --git a/plugins/implementation/skills/implement-dispatch/evals/evals.json b/plugins/implementation/skills/implement-dispatch/evals/evals.json new file mode 100644 index 000000000..8423b9066 --- /dev/null +++ b/plugins/implementation/skills/implement-dispatch/evals/evals.json @@ -0,0 +1,80 @@ +{ + "skill_name": "implement-dispatch", + "evals": [ + { + "id": 1, + "name": "autonomous-dispatches-even-when-plan-is-all-main-window", + "prompt": "/implementation:implement-dispatch — execute the approved plan. This is an autonomous goal-loop run with no human in the turn cycle; the plan's routing table lists every phase as main-window.", + "expected_output": "Detects autonomous orchestration mode and dispatches scope-fenced workers per phase anyway — orchestrated cadence is the default in autonomous mode even when the plan routes all-main-window, so it synthesizes per-phase worker rows rather than doing volume edits from the orchestrator, capping concurrent dispatch at 3-5 workers.", + "files": [], + "expectations": [ + "Detects autonomous mode from the goal-loop / no-human-in-cycle signal", + "Dispatches workers per phase even though the routing table is all main-window — synthesizes worker rows rather than editing inline", + "Does NOT perform the volume edits itself from the orchestrator window in autonomous mode", + "Caps concurrent dispatch at roughly 3-5 workers per wave" + ] + }, + { + "id": 2, + "name": "interactive-all-main-window-hands-back-to-implement", + "prompt": "/implementation:implement-dispatch — this is an interactive session with me reviewing each turn, and the approved plan has no routing table (all phases are main-window).", + "expected_output": "In interactive mode with no worker rows in the routing table, hands execution back to /implement's classic inline cadence instead of dispatching workers — dispatch is only for worker-routed phases here.", + "files": [], + "expectations": [ + "Detects interactive mode (human reviewing each turn) rather than autonomous", + "Hands back to /implement classic inline cadence because the routing table is absent / all main-window", + "Does NOT force worker dispatch onto an interactive all-main-window plan" + ] + }, + { + "id": 3, + "name": "verifies-worker-return-against-direct-evidence", + "prompt": "/implementation:implement-dispatch phase-2 — the worker for phase 2 reported back: 'Done, all files updated and the build is green.' Accept the phase and move on.", + "expected_output": "Treats the worker return as synthesis, not ground truth: promotes its claims to direct evidence (diff read, grep, file Read) and runs /build itself main-side before accepting the edits — it never accepts the worker's green claim as the build signal.", + "files": [], + "expectations": [ + "Does NOT accept the worker's 'build is green' claim as the authoritative build signal", + "Verifies the worker's claims against direct evidence (reads the diff / greps / reads files) before building on them", + "Runs /build itself from the main window rather than trusting the worker's report" + ] + }, + { + "id": 4, + "name": "brief-carries-scope-fence-and-divergence-clause", + "prompt": "/implementation:implement-dispatch phase-1 — compose the brief and dispatch the worker for phase 1.", + "expected_output": "The composed brief carries an explicit scope fence (enumerated ALLOWED and FORBIDDEN files/actions), the verbatim divergence-escalation clause instructing the worker to STOP and report if an assumption proves wrong or it must touch anything FORBIDDEN, the touched project invariants, and the phase acceptance criteria — before dispatching.", + "files": [], + "expectations": [ + "The brief enumerates an explicit scope fence: ALLOWED vs FORBIDDEN files/actions", + "The brief includes the verbatim divergence-escalation clause (STOP and report on a wrong assumption or a FORBIDDEN touch — do not improvise)", + "The brief includes the phase's acceptance criteria and the project invariants the task touches", + "The scope fence and clause are present before dispatch, not added after the worker returns" + ] + }, + { + "id": 5, + "name": "non-interactive-moderate-divergence-takes-conservative-option", + "prompt": "/implementation:implement-dispatch — autonomous run, no human to escalate to. A worker hit a moderate divergence: the plan named a config key that turned out slightly different, and it's asking how to proceed.", + "expected_output": "In a non-interactive run, moderate divergence does not deadlock the run: it picks the conservative option (truest to the plan's intent, smallest blast radius), logs it to a DEVIATIONS.md beside the plan artifact, and keeps going. Major divergence would still STOP even autonomously.", + "files": [], + "expectations": [ + "Does NOT deadlock the autonomous run waiting for a human on a moderate divergence", + "Picks the conservative option (closest to plan intent, smallest blast radius) for the moderate divergence", + "Logs the deviation to a DEVIATIONS.md beside the plan artifact (what was planned, what was done, why, blast radius)", + "Reserves a hard STOP for major divergence (a fundamental assumption wrong), not this moderate one" + ] + }, + { + "id": 6, + "name": "autonomous-fresh-context-verifier-before-done", + "prompt": "/implementation:implement-dispatch — autonomous run. Phase 3's workers have returned and their acceptance criteria look met; mark the phase DONE.", + "expected_output": "In autonomous mode, before marking the phase [DONE] it dispatches a fresh-context verifier subagent that checks the phase's acceptance criteria against the actual diff — a separate agent, never the producing context auditing itself — and surfaces that result in the response before ending the turn. It still runs the full Step 4 phase-boundary ritual.", + "files": [], + "expectations": [ + "Dispatches a separate fresh-context verifier subagent before marking the phase DONE (not the producing context self-auditing)", + "The verifier checks acceptance criteria against the actual diff, and its result is surfaced in the response before the turn ends", + "Runs the full /implement Step 4 phase-boundary ritual (plan marks, handoff, status, commit, resume prompt) regardless of resident-vs-clear" + ] + } + ] +} diff --git a/plugins/implementation/skills/implement/evals/evals.json b/plugins/implementation/skills/implement/evals/evals.json new file mode 100644 index 000000000..fcd791bca --- /dev/null +++ b/plugins/implementation/skills/implement/evals/evals.json @@ -0,0 +1,79 @@ +{ + "skill_name": "implement", + "evals": [ + { + "id": 1, + "name": "bugfix-mode-incremental-cadence", + "prompt": "There's a bug in the login flow — expired tokens are being accepted. Fix it.", + "expected_output": "Detects Bugfix execution mode from the 'fix' signal and consults the bugfix context guidance, runs the Step 1 prerequisite check (plan/branch/dirty-tree) before the first edit, then executes with the incremental cadence: one logical block, build check, test, commit at green — not the whole change before any validation.", + "files": [], + "expectations": [ + "Selects Bugfix execution mode from the 'fix' signal rather than asking when the mode is unambiguous", + "Runs the Step 1 prerequisite check (approved plan, correct branch, clean-enough tree) before writing code", + "Executes incrementally — validates (build/test) after a logical block instead of implementing the entire change before any validation", + "Commits at a green checkpoint rather than leaving the fix uncommitted or committing a red state" + ] + }, + { + "id": 2, + "name": "branch-check-stops-on-default-branch", + "prompt": "/implementation:implement feature — start building the new export endpoint. (Assume the working session is currently on the `main` branch in a PR-based repo.)", + "expected_output": "The Step 1 prerequisite check detects the session is on the default branch in a feature-branch workflow and stops to create a feature branch (git checkout -b, or /source-control:worktree when installed) BEFORE the first edit — it does not begin editing on main.", + "files": [], + "expectations": [ + "Detects that the current branch is the default branch (main/master) in a feature-branch workflow", + "Creates or directs creation of a feature branch before the first code edit, rather than editing on the default branch", + "Does NOT proceed to write code on the default branch and leave branch creation for later" + ] + }, + { + "id": 3, + "name": "major-divergence-routes-back-to-planning", + "prompt": "/implementation:implement — I approved a plan that assumed the vendor SDK exposes a streaming API, but partway through it's clear the SDK only supports batch calls, so the whole approach doesn't fit. Keep going and make it work.", + "expected_output": "Recognizes this as major divergence (a fundamental plan assumption was wrong), STOPS writing code rather than forcing workarounds, runs external research for alternative approaches first, then routes back to the planning skill (/planning:architect review when installed) so the user can approve the new direction — it does not silently push through with hacks.", + "files": [], + "expectations": [ + "Classifies the wrong SDK-capability assumption as major divergence, not a minor inline fixup", + "STOPS writing code instead of pushing through with workarounds/hacks to force the original plan to fit", + "Runs external research for alternatives before re-planning, then routes back to the planning skill for the user to approve the new direction", + "Does NOT silently expand scope or improvise a new design without surfacing it to the user" + ] + }, + { + "id": 4, + "name": "never-declares-impossible-without-alternatives", + "prompt": "/implementation:implement — I tried wiring the webhook two ways and both failed, so this integration is impossible. Confirm that and stop.", + "expected_output": "Does not accept 'impossible' after only two attempts. It presents the two failures as a progress update, not a conclusion, and researches one level deeper (GitHub issues for workaround flags, bypass options, alternative APIs) before escalating — it never declares the task impossible without exhausting alternatives.", + "files": [], + "expectations": [ + "Does NOT declare the integration impossible on the basis of two failed attempts", + "Frames the two failures as a progress update and researches deeper (workaround flags, bypass options, alternative APIs) before concluding", + "Only escalates 'cannot be done' after alternatives are exhausted, not as the default response to early failure" + ] + }, + { + "id": 5, + "name": "over-correction-guard-on-pushback", + "prompt": "/implementation:implement — at this phase boundary you proposed three follow-up actions (two are plan work-items, one you surfaced from an anomaly mid-implementation). I don't like where this is going — drop it.", + "expected_output": "Applies the over-correction guard: with pushback on multiple (>=2) proposed actions it does NOT silently drop all of them. It asks per-category with AskUserQuestion (drop all N / only the invented item / specific items), and it keeps invented follow-ups separated from plan-anticipated work rather than batching them.", + "files": [], + "expectations": [ + "Does NOT silently drop all proposed actions in response to the ambiguous 'drop it' pushback", + "Asks a per-category clarifying question (drop all / only invented / specific items) via AskUserQuestion before dropping anything", + "Keeps the invented/anomaly-surfaced action classified separately from the plan work-items rather than batching them into one undifferentiated proposal" + ] + }, + { + "id": 6, + "name": "autonomous-run-chains-to-implement-dispatch", + "prompt": "/implementation:implement — execute the approved plan. This is an autonomous goal-loop run with no human in the turn cycle and the plan routes phases to worker subagents.", + "expected_output": "Detects orchestration mode (autonomous run / worker-routed phases) and, after the Step 1 prerequisite check passes, chains to /implement-dispatch via the Skill tool for those phases instead of running the inline Step 2 cadence — but Step 1's branch/plan/dirty-tree preflight still runs.", + "files": [], + "expectations": [ + "Detects orchestration mode from the autonomous / worker-routed signal rather than defaulting to inline execution", + "Chains to /implement-dispatch (via the Skill tool) for the worker-routed phases instead of editing inline", + "Still runs the Step 1 prerequisite preflight (branch/plan/dirty-tree) even in the orchestrated path — dispatch does not skip it" + ] + } + ] +} diff --git a/plugins/implementation/skills/lint/evals/evals.json b/plugins/implementation/skills/lint/evals/evals.json new file mode 100644 index 000000000..d397c8b35 --- /dev/null +++ b/plugins/implementation/skills/lint/evals/evals.json @@ -0,0 +1,66 @@ +{ + "skill_name": "lint", + "evals": [ + { + "id": 1, + "name": "check-mode-detects-and-runs-lint-only", + "prompt": "/implementation:lint — I touched a few TypeScript files and a bash script.", + "expected_output": "Auto-detects the affected ecosystems (typescript, bash) from git status, runs each ecosystem's resolved check-cmd in check mode only — reporting violations without modifying files — and presents a per-ecosystem lint/format results table. It runs lint+format, not a full build/test cycle.", + "files": [], + "expectations": [ + "Auto-detects affected ecosystems (typescript, bash) from the changed files", + "Runs in check mode by default — reports violations WITHOUT modifying any files", + "Runs lint/format checks only, not a build or test cycle (that is /build)", + "Presents a per-ecosystem results table with pass / FAIL / skip / — statuses" + ] + }, + { + "id": 2, + "name": "fix-mode-parsed-and-auto-corrects", + "prompt": "/implementation:lint dotnet --fix", + "expected_output": "Parses the --fix flag as fix mode and the dotnet filter, runs the ecosystem's fix-cmd to auto-correct where the linter supports it, and notes which ecosystems were auto-fixed versus which have no auto-fix (fix capability derives from a non-null fix-cmd in the config).", + "files": [], + "expectations": [ + "Parses --fix as fix mode (auto-correct) rather than check mode, and dotnet as the ecosystem filter", + "Runs the resolved fix-cmd to auto-correct where the linter supports it", + "Reports which ecosystems were auto-fixed vs which are check-only (no non-null fix-cmd)" + ] + }, + { + "id": 3, + "name": "owns-yaml-and-cross-cutting-surfaces", + "prompt": "/implementation:lint all — the change set includes a GitHub Actions workflow YAML and touches several text files across the repo.", + "expected_output": "Runs the lint-only `yaml` and `cross-cutting` surfaces that /build does not — yaml for the workflow file, and cross-cutting (typos/gitleaks/editorconfig-checker) when any text file changed and the repo opts into those tools — resolving the editorconfig-checker binary-name variant before substituting it in.", + "files": [], + "expectations": [ + "Runs the `yaml` surface for the workflow file (a /lint-only surface not covered by /build)", + "Runs the `cross-cutting` surface when any text file changed AND the repo opts into its tools", + "Honors each ecosystem's opt-in — does not run cross-cutting tools the repo has not configured" + ] + }, + { + "id": 4, + "name": "missing-tool-skips-file-outside-ecosystem-silent", + "prompt": "/implementation:lint — the diff includes a .png image and some markdown, but markdownlint is not installed.", + "expected_output": "Reports the markdown ecosystem as `skip` with the install-hint (tool missing is a skip, never a FAIL), and silently ignores the .png (a file outside any ecosystem produces no noise) — it does not fail the run for either.", + "files": [], + "expectations": [ + "Reports the ecosystem as `skip` with its install-hint when the linter tool is missing from PATH — not FAIL", + "Silently skips the .png file (outside any ecosystem) without emitting a warning or failure for it", + "Does not fail the overall run because of the missing tool or the unclassified binary file" + ] + }, + { + "id": 5, + "name": "no-changes-directs-to-lint-all", + "prompt": "/implementation:lint — the working tree is clean and there is no branch diff versus the default branch.", + "expected_output": "When neither the working tree nor the branch diff yields changes and no ecosystem filter was given, it reports 'No changes found' and points the user to `/lint all` (full-repo) or `/lint ` — rather than silently running everything or erroring.", + "files": [], + "expectations": [ + "Falls back to the branch diff before concluding there is nothing to lint", + "Reports 'No changes found' and suggests `/lint all` or `/lint ` when both detection paths are empty and no filter was given", + "Does not silently run every ecosystem when no changes and no filter are present" + ] + } + ] +} diff --git a/plugins/implementation/skills/setup/evals/evals.json b/plugins/implementation/skills/setup/evals/evals.json new file mode 100644 index 000000000..a2c412007 --- /dev/null +++ b/plugins/implementation/skills/setup/evals/evals.json @@ -0,0 +1,54 @@ +{ + "skill_name": "setup", + "evals": [ + { + "id": 1, + "name": "infers-then-interviews-before-writing", + "prompt": "/implementation:setup", + "expected_output": "Detects which ecosystems apply from what actually exists in the repo (build files, lockfiles, linter configs), drafts each one's command surface seeded from the bundled portable default and specialized to the repo, then interviews the user one ecosystem at a time with a recommendation — writing nothing until the user accepts.", + "files": [], + "expectations": [ + "Infers candidate ecosystems from the repo layout (project/build files, lockfiles, linter configs) before asking", + "Drafts each ecosystem's command surface seeded from the bundled portable default and specialized to the repo", + "Interviews the user one ecosystem at a time with a recommendation, rather than writing config blind", + "Writes the .claude/ecosystems/*.yaml files only after the user accepts the drafts" + ] + }, + { + "id": 2, + "name": "idempotent-reads-existing-config-first", + "prompt": "/implementation:setup — this repo already has `.claude/ecosystems/dotnet.yaml` and `.claude/ecosystems/python.yaml` from a previous run.", + "expected_output": "Reads the existing ecosystem files first, summarizes each one's current command surface, and proposes changes against that baseline rather than overwriting blind — nothing is dropped without the user confirming.", + "files": [], + "expectations": [ + "Detects and reads the existing `.claude/ecosystems/*.yaml` files before proposing changes", + "Summarizes the current per-ecosystem command surface and proposes edits against that baseline", + "Does not overwrite or drop existing ecosystem config without the user confirming" + ] + }, + { + "id": 3, + "name": "writes-consumer-tracked-config-not-plugin-dir", + "prompt": "/implementation:setup — just write a sensible default ecosystem config for this repo.", + "expected_output": "Materializes the config only inside the consumer's tracked `$REPO_ROOT/.claude/ecosystems/`, confirms each file is tracked (not gitignored), and offers the `.local.yaml` overlay + user-global convention — it NEVER writes configuration into the plugin directory or the plugin data directory.", + "files": [], + "expectations": [ + "Writes ecosystem files only under the consumer's tracked `$REPO_ROOT/.claude/ecosystems/`", + "Does NOT write configuration into the plugin directory or the plugin data directory", + "Confirms the files are tracked (not gitignored) and offers the `.local.yaml` personal-overlay / user-global convention" + ] + }, + { + "id": 4, + "name": "ambiguous-inference-asks-not-guesses", + "prompt": "/implementation:setup typescript — the package.json has both a `test` and a `test:ci` script and no configured linter is obvious.", + "expected_output": "When inference is ambiguous (multiple candidate test runners, no configured linter) it asks the user for the command rather than guessing, and scopes the whole run to just the typescript ecosystem because $ARGUMENTS named it.", + "files": [], + "expectations": [ + "Asks the user which command to use when inference is ambiguous (multiple test scripts / no clear linter) instead of guessing one", + "Scopes the run to only the typescript ecosystem because the argument named it", + "Does not fabricate a test-cmd or check-cmd that the repo does not clearly support" + ] + } + ] +} From 3c84a4419a8ccceb3d348d96caf4c2bd251a43ee Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 12 Jul 2026 21:27:39 -0400 Subject: [PATCH 2/4] chore(evals): address bot-review readability findings Two minor bot-review improvements (no behavioral change): - implement case 1 expected_output now notes the case fires as a model-invoked skill from natural language (no explicit slash prefix), so eval runners understand the intent. - build case 4 expectation 3 drops the unverifiable "/lint redirect" assertion; the verifiable contract is that /build never runs the yaml/cross-cutting surfaces. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/implementation/skills/build/evals/evals.json | 2 +- plugins/implementation/skills/implement/evals/evals.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/implementation/skills/build/evals/evals.json b/plugins/implementation/skills/build/evals/evals.json index 7a5a5d9cb..4d9960e2a 100644 --- a/plugins/implementation/skills/build/evals/evals.json +++ b/plugins/implementation/skills/build/evals/evals.json @@ -47,7 +47,7 @@ "expectations": [ "Runs only the six /build-covered ecosystems (dotnet, python, typescript, bash, powershell, markdown)", "Does NOT run the `yaml` surface for the workflow file — that is /lint-only", - "Does NOT run the `cross-cutting` surface (its `**` glob is never matched by /build); points the user to /lint for those" + "Does NOT run the `cross-cutting` surface — its `**` glob is never matched by /build (both `yaml` and `cross-cutting` are /lint-only surfaces)" ] }, { diff --git a/plugins/implementation/skills/implement/evals/evals.json b/plugins/implementation/skills/implement/evals/evals.json index fcd791bca..72d7478ab 100644 --- a/plugins/implementation/skills/implement/evals/evals.json +++ b/plugins/implementation/skills/implement/evals/evals.json @@ -5,7 +5,7 @@ "id": 1, "name": "bugfix-mode-incremental-cadence", "prompt": "There's a bug in the login flow — expired tokens are being accepted. Fix it.", - "expected_output": "Detects Bugfix execution mode from the 'fix' signal and consults the bugfix context guidance, runs the Step 1 prerequisite check (plan/branch/dirty-tree) before the first edit, then executes with the incremental cadence: one logical block, build check, test, commit at green — not the whole change before any validation.", + "expected_output": "Fires as a model-invoked skill from the natural-language 'fix' signal (no explicit /implementation:implement prefix). Detects Bugfix execution mode and consults the bugfix context guidance, runs the Step 1 prerequisite check (plan/branch/dirty-tree) before the first edit, then executes with the incremental cadence: one logical block, build check, test, commit at green — not the whole change before any validation.", "files": [], "expectations": [ "Selects Bugfix execution mode from the 'fix' signal rather than asking when the mode is unambiguous", From 84ad0ee7e5acaf957ed909001ddc72768d64afac Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 12 Jul 2026 21:31:46 -0400 Subject: [PATCH 3/4] chore(evals): split malformed-config case out of build valid-config eval MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex review (P2): build case 3 sets up a valid tracked dotnet.yaml but one expectation graded malformed-file degrade behavior — a scenario mismatch that could fail a correct valid-config answer. Replace that expectation with a consumer-cmd-wins check, and cover the malformed-degrades-to-inference guardrail in its own dedicated case 6. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../implementation/skills/build/evals/evals.json | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/plugins/implementation/skills/build/evals/evals.json b/plugins/implementation/skills/build/evals/evals.json index 4d9960e2a..d38bde2d0 100644 --- a/plugins/implementation/skills/build/evals/evals.json +++ b/plugins/implementation/skills/build/evals/evals.json @@ -35,7 +35,7 @@ "expectations": [ "Resolves commands via the ladder with the consumer's `.claude/ecosystems/dotnet.yaml` taking precedence (rung 1)", "Uses the bundled portable default only as the rung-4 fallback when no consumer file is present", - "A malformed consumer file degrades to inference with a warning rather than hard-stopping" + "Runs the consumer's declared test-cmd, not the bundled default's" ] }, { @@ -61,6 +61,18 @@ "Classifies and verifies the checkpoint-committed changes rather than reporting nothing to do", "Only reports 'No changes found' when neither the working tree nor the branch diff yields changes and no ecosystem argument was given" ] + }, + { + "id": 6, + "name": "malformed-consumer-config-degrades-not-hard-stops", + "prompt": "/implementation:build dotnet — the repo's tracked `.claude/ecosystems/dotnet.yaml` exists but is malformed (invalid YAML / missing required keys).", + "expected_output": "A malformed consumer ecosystem file warns and degrades to inference (rung 2) rather than hard-stopping — the ladder is tolerant, so a bad consumer file never aborts the build; it still resolves a command surface and runs.", + "files": [], + "expectations": [ + "Warns that the consumer `.claude/ecosystems/dotnet.yaml` is malformed rather than silently ignoring it", + "Degrades to inference (rung 2) to resolve the command surface instead of hard-stopping", + "Does NOT abort the run with a hard error because the consumer file is malformed" + ] } ] } From 26d367d57244be1f480b03aafbbc4564e4935de8 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 12 Jul 2026 21:39:11 -0400 Subject: [PATCH 4/4] chore(evals): make markdownlint opt-in explicit in lint missing-tool case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codex review (P2): lint case 4 expected a missing-tool `skip` for markdown, but /lint honors opt-in first — without a markdownlint config, silently skipping markdown is the correct behavior, which could penalize a valid answer. State in the prompt that the repo has opted into markdownlint (config present) so the missing-tool skip is the unambiguously-graded behavior. Co-Authored-By: Claude Opus 4.8 (1M context) --- plugins/implementation/skills/lint/evals/evals.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plugins/implementation/skills/lint/evals/evals.json b/plugins/implementation/skills/lint/evals/evals.json index d397c8b35..ec3a2cd5a 100644 --- a/plugins/implementation/skills/lint/evals/evals.json +++ b/plugins/implementation/skills/lint/evals/evals.json @@ -41,11 +41,11 @@ { "id": 4, "name": "missing-tool-skips-file-outside-ecosystem-silent", - "prompt": "/implementation:lint — the diff includes a .png image and some markdown, but markdownlint is not installed.", - "expected_output": "Reports the markdown ecosystem as `skip` with the install-hint (tool missing is a skip, never a FAIL), and silently ignores the .png (a file outside any ecosystem produces no noise) — it does not fail the run for either.", + "prompt": "/implementation:lint — the repo has a markdownlint config (the markdown surface is opted in), and the diff includes a .png image plus some markdown, but the markdownlint tool is not on PATH.", + "expected_output": "Because the markdown surface is opted in (config present) but the tool is missing, reports the markdown ecosystem as `skip` with the install-hint (tool missing is a skip, never a FAIL), and silently ignores the .png (a file outside any ecosystem produces no noise) — it does not fail the run for either.", "files": [], "expectations": [ - "Reports the ecosystem as `skip` with its install-hint when the linter tool is missing from PATH — not FAIL", + "Reports the (opted-in) markdown ecosystem as `skip` with its install-hint when the markdownlint tool is missing from PATH — not FAIL", "Silently skips the .png file (outside any ecosystem) without emitting a warning or failure for it", "Does not fail the overall run because of the missing tool or the unclassified binary file" ]