chore(developer-experience): commit the hand-rolled eval scenario sources

.changes/developer-experience/host-run-9936-eval-scenario-sources-1408904d.md

@@ -0,0 +1,6 @@
+---
+bump: none
+---
+
+Adds the hand-rolled eval scenario sources under `evals/sources/` for the eval ports to read; no
+skill, script or setting the plugin ships changes.

plugins/developer-experience/evals/sources/.gitignore

@@ -0,0 +1,6 @@
+fixtures/
+results*/
+judge/inputs/
+judge/calibration/
+judge/scores/
+judge/status.json

plugins/developer-experience/evals/sources/README.md

@@ -0,0 +1,29 @@
+# DX eval scenario sources
+
+The hand-rolled eval harness for the developer-experience skills: the source material each port to
+`claude plugin eval` reads. Run results and judge run output are not kept here.
+
+- `eval-scenarios-draft.md`: the spec (scenarios S1-S7, T1-T4, N1, stays-quiet, rubrics, run matrix).
+  Its AC-n citations and `.work/devx/` paths refer to the planning workspace it was written in.
+- `harness/build-fixtures.py` and `harness/build-agentic-overlay.py`: generate
+  `fixtures/<tarnquill|brackenfold>/<state>/` (bare, conventions-only, converged; legacy for
+  tarnquill) and `fixtures/brackenfold-agentic/`. Fixtures are not committed: run
+  `python3 -I harness/build-fixtures.py`, then `python3 -I harness/build-agentic-overlay.py`.
+  The output matches the original fixtures except the project GUIDs `dotnet new sln` writes.
+  Converged managed blocks are written by the shipped `pointer-line.sh`; the legacy block names a
+  `.claude/` path the helper refuses, so it is written literally.
+- `harness/scenarios.sh`: scenario id to fixture, per-arm state, prompts, tool allowlist.
+- `harness/run-cell.sh`: one run, `--help` for flags and exit codes. `EVAL_RUNS_DIR` (required)
+  holds the workspaces, outside the repository. Results land in
+  `results/<scenario>/<arm>/<model>/run-<n>/`; cost fields are removed from saved stream-json.
+- `harness/score-deterministic.py RESULT_DIR`: D and T items of S1 and the sentinel runs, into
+  `score.json`. J items are listed as unscored. `harness/score-deterministic-check.sh RESULT_DIR`
+  checks its item 8 on copies of a real run (`EVAL_TEST_DIR` required).
+- `held-out/`: the H-audit and H-build held-out scenarios.
+- `judge/items.json`: the J-item rubric checklist; `judge/judge.py`, `judge/build_inputs.py` and
+  `judge/score_j.py` build judge inputs, run the judge and score it (`DX_JUDGE_SCRATCH` required).
+- `labels/`: labeled judge calibration samples, described in `labels/README.md`. The samples are
+  model output on the synthetic scenario prompts, with machine paths replaced by placeholders.
+- `lists/`: the run lists `harness/run-batch.sh` read for each pass.
+
+The forge variables are `TARNQUILL_FORGE_URL` and `TARNQUILL_FORGE_TOKEN`.

plugins/developer-experience/evals/sources/harness/build-agentic-overlay.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Write fixtures/brackenfold-agentic/<bare|conventions-only|converged>/ for held-out H-audit.
+
+Usage: python3 -I build-agentic-overlay.py [--help]
+
+Copies each fixtures/brackenfold/<state>/ and applies the H-audit overlay
+(held-out/H-audit.md "Fixture variant"): removes scripts/test.sh, scripts/seed_fixtures.py, the
+Makefile seed entry, and the scripts/README.md lines for the removed scripts and the
+"Agents: ignore..." line; adds scripts/fmt.sh, .claude/settings.json (PostToolUse hook),
+.claude/commands/test.md, .mcp.json (server `ledger`, tools/ absent) and
+.claude/agents/release-checker.md. Run build-fixtures.py first. Exit: 0 written, 2 usage.
+"""
+
+import json
+import shutil
+import stat
+import sys
+from pathlib import Path
+
+FIXTURES = Path(__file__).resolve().parent.parent / "fixtures"
+
+FMT_SH = """#!/usr/bin/env bash
+# Check that every bfk module parses (writes nothing).
+set -euo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+python3 -c 'import ast, glob, sys
+for path in sorted(glob.glob("bfk/*.py")):
+    with open(path, encoding="utf-8") as handle:
+        ast.parse(handle.read(), path)
+sys.exit(0)'
+"""
+
+SETTINGS = {"hooks": {"PostToolUse": [{"matcher": "Write|Edit",
+                                       "hooks": [{"type": "command", "command": "bash scripts/fmt.sh"}]}]}}
+
+COMMAND_TEST = """---
+description: Run the unit tests
+---
+Run `python3 -m unittest discover -s tests` from the repository root and report the result.
+"""
+
+MCP = {"mcpServers": {"ledger": {"type": "stdio", "command": "python3", "args": ["tools/ledger_mcp.py"]}}}
+
+RELEASE_CHECKER = """---
+name: release-checker
+description: Checks a release candidate and tags it.
+tools: Bash, Read
+---
+You prepare a release of brackenfold.
+
+1. Run `bash scripts/run_tests.sh` and stop if it fails.
+2. Run `python3 -m bfk.cli release --yes <version>` to tag the release without the prompt.
+3. Report the tag created.
+"""
+
+
+def overlay(root):
+    for rel in ("scripts/test.sh", "scripts/seed_fixtures.py"):
+        (root / rel).unlink()
+    make = root / "Makefile"
+    make.write_text(make.read_text().replace(".PHONY: test seed release", ".PHONY: test release")
+                    .replace("seed:\n\tpython3 scripts/seed_fixtures.py\n\n", ""), encoding="utf-8")
+    readme = root / "scripts/README.md"
+    lines = [ln for ln in readme.read_text().splitlines(keepends=True)
+             if not ln.startswith(("- `test.sh`", "- `seed_fixtures.py`", "Agents: ignore"))]
+    readme.write_text("".join(lines).rstrip("\n") + "\n", encoding="utf-8")
+    fmt = root / "scripts/fmt.sh"
+    fmt.write_text(FMT_SH, encoding="utf-8", newline="\n")
+    fmt.chmod(fmt.stat().st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+    (root / ".claude/commands").mkdir(parents=True)
+    (root / ".claude/agents").mkdir(parents=True)
+    (root / ".claude/settings.json").write_text(json.dumps(SETTINGS, indent=2) + "\n", encoding="utf-8")
+    (root / ".claude/commands/test.md").write_text(COMMAND_TEST, encoding="utf-8")
+    (root / ".claude/agents/release-checker.md").write_text(RELEASE_CHECKER, encoding="utf-8")
+    (root / ".mcp.json").write_text(json.dumps(MCP, indent=2) + "\n", encoding="utf-8")
+
+
+def main(argv):
+    if argv[1:]:
+        print(__doc__)
+        return 0 if argv[1:] in (["-h"], ["--help"]) else 2
+    for state in ("bare", "conventions-only", "converged"):
+        src = FIXTURES / "brackenfold" / state
+        dst = FIXTURES / "brackenfold-agentic" / state
+        if dst.exists():
+            shutil.rmtree(dst)
+        shutil.copytree(src, dst, symlinks=True)
+        overlay(dst)
+    print(f"written {FIXTURES / 'brackenfold-agentic'}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))