feat(ci): run plugin eval suites from dispatch and an opt-in PR label

.github/actions/resolve-config/callers.test.mjs

@@ -6651,7 +6651,10 @@ test("every workflow npm ci reads npm-fetch.npmrc from the job's checkout", () =
       const steps = job.steps ?? [];
       const first = steps.findIndex(installs);
       if (first < 0) continue;
-      const checkout = steps.slice(0, first).findLast((step) => /^actions\/checkout@/.test(step.uses ?? ""));
+      // A pull request head checkout is never the source: its npmrc would let the PR set npm's config.
+      const checkout = steps
+        .slice(0, first)
+        .findLast((step) => /^actions\/checkout@/.test(step.uses ?? "") && !/pull_request\.head\./.test(step.with?.ref ?? ""));
       assert.ok(checkout, `${file} ${id}: no checkout before its npm ci`);
       assert.equal(checkout.with?.["sparse-checkout"], undefined, `${file} ${id}: a sparse checkout may lack the file`);
       const dir = checkout.with?.path ? `${checkout.with.path}/` : "";

.github/lane-funding.json

@@ -111,7 +111,7 @@
       "keyed": false,
       "category": "maintenance",
       "lanes": [
-        { "id": "eval", "workflow": "eval-browser-dispatch.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] }
+        { "id": "eval", "workflow": "eval-browser-dispatch.yml", "launcher": "claude-cli", "triggers": ["workflow_dispatch", "pull_request"], "currentSecrets": ["MAINTENANCE_CLAUDE_CODE_OAUTH_TOKEN"] }
       ]
     }
   ]

.github/scripts/plugin-eval/deterministic-suite.mjs

@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+// deterministic-suite.mjs: copy a plugin's eval suite without the cases that
+// need a paid judge, so a run spends nothing on grading.
+//
+//   deterministic-suite.mjs <plugin-dir> <out-name>
+//
+// Reads the plugin's eval dir (the manifest's experimental.evals, else
+// evals/) and writes <plugin-dir>/<out-name>/ holding every file of it except
+// results/ and the cases with an llm or baseline grader. `claude plugin eval
+// --eval-dir <out-name>` then runs only the deterministic cases. Prints
+// {"evalDir", "kept", "dropped"} as JSON, case paths relative to the eval dir.
+//
+// A case is a directory holding prompt.md or case.yaml, found the way
+// /evals:validate's validate-cases.py finds one. It is dropped when its
+// case.yaml or any graders/*.md names type llm or baseline anywhere, quoted
+// or not: run after that validator passes, every grader has one of the six
+// types, so a case with no such line has deterministic graders only, and a
+// stray match only drops a case, never keeps a paid one.
+//
+// Exit 0 written; 2 a usage error, or the eval dir or output cannot be used.
+import { cpSync, existsSync, readFileSync, readdirSync, rmSync, statSync } from "node:fs";
+import { join, relative, sep } from "node:path";
+import process from "node:process";
+import { pathToFileURL } from "node:url";
+
+const SKIP_DIRS = new Set(["results", "mocks", "graders", "node_modules", "__pycache__"]);
+const PAID = /\btype["']?\s*:\s*["']?(?:llm|baseline)\b/i;
+const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+
+export function evalDirOf(pluginDir) {
+  try {
+    const manifest = JSON.parse(readFileSync(join(pluginDir, ".claude-plugin", "plugin.json"), "utf8"));
+    const named = manifest?.experimental?.evals;
+    if (typeof named === "string" && NAME.test(named)) {
+      return named;
+    }
+  } catch {
+    // No readable manifest: the default eval dir.
+  }
+  return "evals";
+}
+
+export function findCases(evalDir) {
+  const cases = [];
+  const walk = (dir) => {
+    for (const entry of readdirSync(dir, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
+      if (!entry.isDirectory() || SKIP_DIRS.has(entry.name) || entry.name.startsWith(".")) {
+        continue;
+      }
+      const path = join(dir, entry.name);
+      if (existsSync(join(path, "prompt.md")) || existsSync(join(path, "case.yaml"))) {
+        cases.push(path);
+      } else {
+        walk(path);
+      }
+    }
+  };
+  walk(evalDir);
+  return cases;
+}
+
+export function needsJudge(caseDir) {
+  const files = [join(caseDir, "case.yaml")];
+  const graders = join(caseDir, "graders");
+  if (existsSync(graders)) {
+    files.push(...readdirSync(graders).filter((name) => name.endsWith(".md")).map((name) => join(graders, name)));
+  }
+  return files.some((file) => existsSync(file) && PAID.test(readFileSync(file, "utf8")));
+}
+
+export function deterministicSuite(pluginDir, outName) {
+  const name = evalDirOf(pluginDir);
+  const evalDir = join(pluginDir, name);
+  const out = join(pluginDir, outName);
+  const kept = [];
+  const dropped = [];
+  for (const caseDir of findCases(evalDir)) {
+    (needsJudge(caseDir) ? dropped : kept).push(relative(evalDir, caseDir).split(sep).join("/"));
+  }
+  const skip = new Set([join(evalDir, "results"), ...dropped.map((path) => join(evalDir, path))]);
+  rmSync(out, { recursive: true, force: true });
+  cpSync(evalDir, out, { recursive: true, filter: (source) => !skip.has(source) });
+  return { evalDir: name, kept, dropped };
+}
+
+export function main(argv = process.argv.slice(2)) {
+  const [pluginDir, outName] = argv;
+  if (argv.length !== 2 || !NAME.test(outName ?? "")) {
+    console.error("usage: deterministic-suite.mjs <plugin-dir> <out-name>");
+    return 2;
+  }
+  const evalDir = join(pluginDir, evalDirOf(pluginDir));
+  if (!existsSync(evalDir) || !statSync(evalDir).isDirectory()) {
+    console.error(`deterministic-suite: ${evalDir} is not a directory`);
+    return 2;
+  }
+  if (evalDirOf(pluginDir) === outName) {
+    console.error("deterministic-suite: the output would replace the eval dir");
+    return 2;
+  }
+  try {
+    console.log(JSON.stringify(deterministicSuite(pluginDir, outName)));
+  } catch (error) {
+    console.error(`deterministic-suite: ${error.message}`);
+    return 2;
+  }
+  return 0;
+}
+
+if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
+  process.exitCode = main();
+}

.github/scripts/plugin-eval/deterministic-suite.test.mjs

@@ -0,0 +1,77 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { test } from "node:test";
+import { fileURLToPath } from "node:url";
+
+import { deterministicSuite } from "./deterministic-suite.mjs";
+
+const SCRIPT = join(dirname(fileURLToPath(import.meta.url)), "deterministic-suite.mjs");
+
+function withPlugin(files, run) {
+  const dir = mkdtempSync(join(tmpdir(), "det-suite-"));
+  try {
+    for (const [path, text] of Object.entries(files)) {
+      mkdirSync(dirname(join(dir, path)), { recursive: true });
+      writeFileSync(join(dir, path), text);
+    }
+    return run(dir);
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+const SUITE = {
+  "evals/regex-only/prompt.md": "Say hi.\n",
+  "evals/regex-only/graders/says-hi.md": "---\ntype: regex\npattern: hi\n---\n",
+  "evals/judged/prompt.md": "Explain.\n",
+  "evals/judged/graders/clear.md": "---\ntype: llm\n---\nThe answer is clear.\n",
+  "evals/listed/case.yaml": 'schema_version: "1"\nname: listed\ngraders:\n  - {name: same, type: "baseline", baseline_file: b.md, criteria: x}\n',
+  "evals/group/nested/prompt.md": "Use the tool.\n",
+  "evals/group/nested/graders/used.md": "---\ntype: tool_used\ntool: Skill\n---\n",
+  "evals/mocks/server.json": "{}\n",
+  "evals/results/old/aggregate-result.json": "{}\n",
+};
+
+test("only the cases with deterministic graders are copied, with the suite's other files", () => {
+  withPlugin(SUITE, (dir) => {
+    const { evalDir, kept, dropped } = deterministicSuite(dir, "evals-deterministic");
+    assert.equal(evalDir, "evals");
+    assert.deepEqual(kept, ["group/nested", "regex-only"]);
+    assert.deepEqual(dropped.sort(), ["judged", "listed"]);
+    const out = join(dir, "evals-deterministic");
+    assert.ok(existsSync(join(out, "regex-only/graders/says-hi.md")));
+    assert.ok(existsSync(join(out, "group/nested/graders/used.md")));
+    assert.ok(existsSync(join(out, "mocks/server.json")));
+    assert.ok(!existsSync(join(out, "judged")));
+    assert.ok(!existsSync(join(out, "listed")));
+    assert.ok(!existsSync(join(out, "results")));
+  });
+});
+
+test("the manifest's eval dir is read", () => {
+  withPlugin(
+    {
+      ".claude-plugin/plugin.json": JSON.stringify({ name: "p", experimental: { evals: "suite" } }),
+      "suite/one/prompt.md": "Hi.\n",
+      "suite/one/graders/g.md": "---\ntype: file_exists\npath: x\n---\n",
+    },
+    (dir) => {
+      assert.deepEqual(deterministicSuite(dir, "evals-deterministic"), { evalDir: "suite", kept: ["one"], dropped: [] });
+    },
+  );
+});
+
+test("the command prints the split as JSON and refuses a missing eval dir or a bad name", () => {
+  withPlugin(SUITE, (dir) => {
+    const ok = spawnSync(process.execPath, [SCRIPT, dir, "evals-deterministic"], { encoding: "utf8" });
+    assert.equal(ok.status, 0, ok.stderr);
+    assert.deepEqual(JSON.parse(ok.stdout).kept, ["group/nested", "regex-only"]);
+    assert.equal(spawnSync(process.execPath, [SCRIPT, dir, "../x"], { encoding: "utf8" }).status, 2);
+    assert.equal(spawnSync(process.execPath, [SCRIPT, dir, "evals"], { encoding: "utf8" }).status, 2);
+    assert.equal(spawnSync(process.execPath, [SCRIPT, join(dir, "absent"), "evals-deterministic"], { encoding: "utf8" }).status, 2);
+    assert.equal(readFileSync(join(dir, "evals/judged/prompt.md"), "utf8"), "Explain.\n");
+  });
+});

.github/scripts/plugin-eval/fixtures/errored.json

@@ -0,0 +1,123 @@
+{
+  "schemaVersion": 1,
+  "claudeVersion": "2.1.310",
+  "costUsd": 0.42,
+  "partial": false,
+  "suite": {
+    "root": "plugins/testing",
+    "ablation": "with-without",
+    "threshold": 0.8,
+    "concurrency": 1,
+    "plugins": [
+      "plugins/testing"
+    ]
+  },
+  "aggregates": {
+    "casesTotal": 1,
+    "casesPassed": 1,
+    "overallScore": 1,
+    "meanDelta": 0.5
+  },
+  "cases": [
+    {
+      "name": "writes-a-test",
+      "dir": "writes-a-test",
+      "runsPerCase": 2,
+      "arms": {
+        "with": [
+          {
+            "score": 1,
+            "passed": true,
+            "error": "timeout",
+            "skippedPaidGraders": false,
+            "costUsd": 0.1,
+            "graders": [
+              {
+                "name": "fires",
+                "passed": true,
+                "withOnly": true,
+                "scored": false
+              },
+              {
+                "name": "names-file",
+                "passed": true,
+                "scored": true
+              }
+            ],
+            "tracePath": "traces/with-fired.jsonl"
+          },
+          {
+            "score": 1,
+            "passed": true,
+            "error": null,
+            "skippedPaidGraders": false,
+            "costUsd": 0.1,
+            "graders": [
+              {
+                "name": "fires",
+                "passed": true,
+                "withOnly": true,
+                "scored": false
+              },
+              {
+                "name": "names-file",
+                "passed": true,
+                "scored": true
+              }
+            ],
+            "tracePath": "traces/with-unfired.jsonl"
+          }
+        ],
+        "without": [
+          {
+            "score": 0.5,
+            "passed": false,
+            "error": null,
+            "skippedPaidGraders": false,
+            "costUsd": 0.1,
+            "graders": [
+              {
+                "name": "fires",
+                "passed": true,
+                "withOnly": true,
+                "scored": false
+              },
+              {
+                "name": "names-file",
+                "passed": false,
+                "scored": true
+              }
+            ]
+          },
+          {
+            "score": 0.5,
+            "passed": false,
+            "error": null,
+            "skippedPaidGraders": false,
+            "costUsd": 0.1,
+            "graders": [
+              {
+                "name": "fires",
+                "passed": true,
+                "withOnly": true,
+                "scored": false
+              },
+              {
+                "name": "names-file",
+                "passed": false,
+                "scored": true
+              }
+            ]
+          }
+        ]
+      },
+      "aggregates": {
+        "score": 1,
+        "passRate": 1,
+        "scoreWithout": 0.5,
+        "passRateWithout": 0,
+        "delta": 0.5
+      }
+    }
+  ]
+}