kyle-sexton · Oct 18, 2026 · Oct 18, 2026 · Oct 18, 2026 · Oct 18, 2026 · Oct 18, 2026
diff --git a/.changes/source-control/fix-pr-unblock-eval-case-scope-ed5fddbc.md b/.changes/source-control/fix-pr-unblock-eval-case-scope-ed5fddbc.md
@@ -0,0 +1,7 @@
+---
+bump: patch
+---
+
+### Fixed
+
+- The `comments --ci` review-thread tool's `check-paths` and `record` now also allow a file in the `plugins/<p>/` directory a kept thread's path sits in, except one under `hooks/`, `.claude-plugin/` or a `.claude/` directory, a `CLAUDE.md` or `AGENTS.md`, or a `.sh`, `.py`, `.mjs`, `.js` or `.ts` file, and a test file (`test_*.py`, `*_test.py`, `*.test.*`, `*.spec.*`) in the same directory as a kept thread's path. `.github/` stays refused.
@@ -22,6 +22,8 @@
 import { appendFileSync, readFileSync, writeFileSync } from "node:fs";
 import { pathToFileURL } from "node:url";
 
+import { isProtectedPath } from "./thread-scope.mjs";
+
 // The round cap is the config's `loop.review-rounds`, passed in as
 // REVIEW_ROUNDS; the resolver's default is the fallback.
 export const DEFAULT_MAX_ROUNDS = 2;
@@ -39,10 +41,6 @@ const FILES_CAP = 3000;
 
 const firstLine = (body) => String(body ?? "").split("\n")[0].replace(/\r$/, "");
 
-export function isProtectedPath(path) {
-  return /^\.github(\/|$)/i.test(String(path));
-}
-
 // Round markers the lanes App posted; any other author's copy is ignored.
 export const isRoundMarker = (comment, appId) => comment.user?.id === appId && firstLine(comment.body) === ROUND_MARKER;
 

@@ -0,0 +1,112 @@
+// Runs the review-threads path check of pr-run-activity-write.yml's promote
+// step (the `script:` of the step with `id: promote`) against fake GitHub
+// clients. A changed path in scope reaches the next gate, the hold-label read,
+// which fails here because the case passes no hold labels.
+import assert from "node:assert/strict";
+import { createHash } from "node:crypto";
+import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { createRequire } from "node:module";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { after, test } from "node:test";
+
+const WORKFLOW = new URL("../../workflows/pr-run-activity-write.yml", import.meta.url);
+const SCOPE = new URL("./thread-scope.mjs", import.meta.url);
+const HEAD = "a".repeat(40);
+const LANE = "b".repeat(40);
+
+function promoteScript() {
+  const lines = readFileSync(WORKFLOW, "utf8").split("\n");
+  const id = lines.findIndex((line) => /^\s+id: promote$/.test(line));
+  assert.notEqual(id, -1, "the workflow has the promote step");
+  const start = lines.findIndex((line, i) => i > id && /^\s+script: \|$/.test(line));
+  const keyIndent = lines[start].search(/\S/);
+  const body = [];
+  for (const line of lines.slice(start + 1)) {
+    if (line.trim() !== "" && line.search(/\S/) <= keyIndent) break;
+    body.push(line);
+  }
+  const indent = Math.min(...body.filter((l) => l.trim()).map((l) => l.search(/\S/)));
+  return body.map((l) => l.slice(indent)).join("\n");
+}
+
+const dir = mkdtempSync(join(tmpdir(), "promote-scope-"));
+after(() => rmSync(dir, { recursive: true, force: true }));
+
+async function promote(files, { scope = readFileSync(SCOPE), sha256 } = {}) {
+  const threadFile = join(dir, "threads.json");
+  writeFileSync(threadFile, JSON.stringify({ threads: [{ path: "plugins/p/evals/c/case.yaml" }, { path: "src/round.py" }] }));
+  const scopeScript = join(dir, "thread-scope.mjs");
+  writeFileSync(scopeScript, scope);
+  const env = {
+    HEAD_SHA: HEAD,
+    HEAD_REF: "feature",
+    LANE_BRANCH: "lane-commits/x",
+    MERGE_BASE: "false",
+    THREAD_FILE: threadFile,
+    SCOPE_SCRIPT: scopeScript,
+    SCOPE_SHA256: sha256 ?? createHash("sha256").update(readFileSync(SCOPE)).digest("hex"),
+    DEPENDENCY_REVIEW: "false",
+    HOLD_LABELS: "",
+  };
+  const saved = { ...process.env };
+  Object.assign(process.env, env);
+  const failed = [];
+  const core = { setOutput() {}, info() {}, warning() {}, setFailed: (message) => failed.push(message) };
+  const github = {
+    request: async () => ({ data: { object: { sha: LANE } } }),
+    rest: {
+      repos: { compareCommitsWithBasehead: async () => ({ data: { files } }) },
+      git: { deleteRef: async () => {} },
+    },
+  };
+  const context = { repo: { owner: "o", repo: "r" } };
+  try {
+    const AsyncFunction = (async () => {}).constructor;
+    await new AsyncFunction("github", "context", "core", "require", promoteScript())(
+      github,
+      context,
+      core,
+      createRequire(import.meta.url),
+    );
+  } finally {
+    process.env = saved;
+  }
+  return failed;
+}
+
+const HOLD = /^No hold labels were read/;
+const OUT = /^The lane branch changes a path outside the kept threads' scope or under \.github\//;
+
+test("in-scope changes pass the path check: the thread's path, its plugin's files, a sibling test", async () => {
+  const failed = await promote([
+    { filename: "plugins/p/evals/c/case.yaml" },
+    { filename: "plugins/p/evals/c/samples/deep/new.json" },
+    { filename: "plugins/p/evals/other/samples/new.json" },
+    { filename: "plugins/p/README.md" },
+    { filename: "src/test_round.py" },
+  ]);
+  assert.equal(failed.length, 1);
+  assert.match(failed[0], HOLD);
+});
+
+for (const file of [
+  { filename: "plugins/q/evals/c/samples/new.json" },
+  { filename: "src/lib/test_round.py" },
+  { filename: ".github/workflows/x.yml" },
+  { filename: "plugins/p/evals/c/x.json", previous_filename: ".github/x.yml" },
+]) {
+  test(`an out-of-scope change stops the move: ${file.previous_filename ?? file.filename}`, async () => {
+    const failed = await promote([{ filename: "plugins/p/evals/c/case.yaml" }, file]);
+    assert.equal(failed.length, 1);
+    assert.match(failed[0], OUT);
+  });
+}
+
+test("a rewritten scope copy, or no recorded hash, stops the move", async () => {
+  const permissive = "export const inThreadScope = () => true;\n";
+  for (const options of [{ scope: permissive }, { sha256: "" }]) {
+    const failed = await promote([{ filename: ".github/workflows/x.yml" }], options);
+    assert.deepEqual(failed, ["thread-scope.mjs does not match the base's copy; the PR branch is not moved"]);
+  }
+});
@@ -10,7 +10,8 @@
 //      fields, within the limits below. Any failure posts no reply.
 //   2. For VALID replies, confirms every fix commit reached the PR branch
 //      after the model job's gated head and that the branch changed no path
-//      but the kept threads' and none under .github/.
+//      outside the kept threads' scope (thread-scope.mjs) and none under
+//      .github/.
 //   3. Re-reads each thread and skips one that is resolved, moved, or has a
 //      comment the model job did not see; posts the templated reply on its
 //      root comment and resolves it only when it is VALID. The PR head is
@@ -46,7 +47,6 @@ import {
   countRounds,
   createGitHubApi,
   currentThreads,
-  isProtectedPath,
   isRoundMarker,
   DEFAULT_MAX_ROUNDS,
   maxRoundsFrom,
@@ -57,6 +57,7 @@ import {
   SHA,
   trustedHumansFrom,
 } from "./address-feedback.mjs";
+import { inThreadScope } from "./thread-scope.mjs";
 
 export const LIMITS = { bytes: 262144, note: 2000, evidence: 10, files: 300 };
 const THREAD_ID = /^PRRT_[A-Za-z0-9_-]{1,100}$/;
@@ -168,9 +169,10 @@ export function renderReply({ disposition, evidence, fixSha, note }) {
 }
 
 // Whenever the PR branch moved since the model job's gated head, or a reply is
-// VALID, that range may change only the kept threads' paths (run.json, whose
-// SHA-256 the model job's runner recorded), and nothing under .github/. Every VALID
-// fix must be a commit in that range that changes its own thread's path.
+// VALID, that range may change only what inThreadScope allows around the kept
+// threads' paths (run.json, whose SHA-256 the model job's runner recorded), and
+// nothing under .github/. Every VALID fix must be a commit in that range that
+// changes its own thread's path.
 async function fixesLanded({ api, run, headSha, replies }) {
   const valid = replies.filter((reply) => reply.disposition === "VALID");
   if (valid.length === 0 && run.head_sha === headSha) {
@@ -187,7 +189,7 @@ async function fixesLanded({ api, run, headSha, replies }) {
     files.length < LIMITS.files &&
     files.every((file) =>
       [file.filename, file.previous_filename].every(
-        (path) => path === undefined || (paths.has(path) && !isProtectedPath(path)),
+        (path) => path === undefined || inThreadScope(path, paths),
       ),
     );
   if (!inRange) {

@@ -422,6 +422,23 @@ for (const [label, compare] of [
   });
 }
 
+test("a VALID fix that also adds a file in its thread's plugin lands; another plugin's file does not", async () => {
+  const caseRun = JSON.stringify({ ...RUN, threads: [{ ...RUN.threads[0], path: "plugins/p/evals/c/case.yaml" }] });
+  for (const [sample, landed] of [
+    ["plugins/p/evals/c/samples/new.json", true],
+    ["plugins/p/evals/other/samples/new.json", true],
+    ["plugins/q/evals/c/samples/new.json", false],
+  ]) {
+    const api = fakeApi({
+      compare: { status: "ahead", commits: [{ sha: FIX }], files: [{ filename: "plugins/p/evals/c/case.yaml" }, { filename: sample }] },
+      commits: { [FIX]: { files: [{ filename: "plugins/p/evals/c/case.yaml" }, { filename: sample }] } },
+    });
+    const result = await run(api, lines(reply({ disposition: "VALID", fix_sha: FIX })), caseRun);
+    assert.equal(result.escalationReason, landed ? "" : "feedback-not-landed", sample);
+    assert.equal(api.posted().length, landed ? 1 : 0, sample);
+  }
+});
+
 test("feedback-not-landed, nothing posted: a VALID fix commit that does not change its own thread's path", async () => {
   const api = fakeApi({
     compare: { status: "ahead", commits: [{ sha: FIX }], files: [{ filename: "docs/b.md" }] },

@@ -0,0 +1,41 @@
+// Which paths an address-feedback fix may change, shared by the promote step
+// of .github/workflows/pr-run-activity-write.yml (which imports a SHA-256
+// pinned copy, so it imports nothing itself) and publish-feedback.mjs.
+
+export function isProtectedPath(path) {
+  return /^\.github(\/|$)/i.test(String(path));
+}
+
+const PLUGIN_DIR = /^plugins\/[^/]+\//;
+// In the plugin directory but run or loaded as instructions by the harness or
+// a shell, so out of the plugin-wide part: hooks, manifests, script or code
+// files, and agent rules (CLAUDE.md, AGENTS.md, anything under .claude/).
+const PLUGIN_CODE =
+  /^plugins\/[^/]+\/(hooks|\.claude-plugin)\/|\.(sh|py|mjs|js|ts)$|(^|\/)(claude|agents)\.md$|(^|\/)\.claude\//i;
+const TEST_FILE = /^(test_[^/]*\.py|[^/]*_test\.py|[^/]*\.(test|spec)\.[^/]+)$/;
+const dirOf = (path) => path.slice(0, path.lastIndexOf("/") + 1);
+
+// An empty, . or .. segment, so a leading or doubled / too.
+const BAD_SEGMENT = /(^|\/)\.{0,2}(\/|$)/;
+
+// A kept thread's own path, a file in the plugins/<p>/ directory a kept
+// thread's path sits in that is not under hooks/, .claude-plugin/ or a
+// .claude/ directory, not a CLAUDE.md or AGENTS.md, and not a .sh, .py, .mjs,
+// .js or .ts file, or a test file in the same directory as a
+// kept thread's path; never anything under .github/, and never a path that is
+// not in normal form.
+export function inThreadScope(path, threadPaths) {
+  if (isProtectedPath(path) || BAD_SEGMENT.test(path)) {
+    return false;
+  }
+  if (threadPaths.has(path)) {
+    return true;
+  }
+  const kept = [...threadPaths];
+  const plugin = PLUGIN_DIR.exec(path)?.[0];
+  if (plugin !== undefined && !PLUGIN_CODE.test(path) && kept.some((p) => p.startsWith(plugin))) {
+    return true;
+  }
+  const dir = dirOf(path);
+  return TEST_FILE.test(path.slice(dir.length)) && kept.some((p) => dirOf(p) === dir);
+}
@@ -0,0 +1,131 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+
+import { inThreadScope, isProtectedPath } from "./thread-scope.mjs";
+
+const kept = new Set(["plugins/p/evals/c/graders/x.md", "src/a.js"]);
+
+test("a kept thread's own path is in scope", () => {
+  assert.equal(inThreadScope("src/a.js", kept), true);
+  assert.equal(inThreadScope("plugins/p/evals/c/graders/x.md", kept), true);
+});
+
+// plugins/planning/README.md is the file #8178's run 39117146913 refused.
+test("any file in the same plugins/<p>/ directory as a kept thread's path is in scope, at any depth", () => {
+  for (const path of [
+    "plugins/p/evals/c/samples/y.json",
+    "plugins/p/evals/other/samples/deep/z.json",
+    "plugins/p/evals/evals.json",
+    "plugins/p/skills/s/SKILL.md",
+    "plugins/p/README.md",
+  ]) {
+    assert.equal(inThreadScope(path, kept), true, path);
+  }
+  assert.equal(inThreadScope("plugins/planning/README.md", new Set(["plugins/planning/surface/round.py"])), true);
+});
+
+// Each refused path and the kept thread paths of the four recorded trips.
+test("the four recorded trips are in scope", () => {
+  const trips = [
+    // #8043, run 39100052790: an eval sample beside the thread's case.
+    ["plugins/session-flow/evals/rr-skill-fit-names-owner/samples/names-skill-and-plugin.json", ["plugins/session-flow/evals/rr-skill-fit-names-owner/case.yaml"]],
+    // #8103, run 39107762307: the thread is on case.yaml:18.
+    ["plugins/discipline/evals/ror-second-error-way/samples/second-way-named.json", ["plugins/discipline/evals/ror-second-error-way/case.yaml"]],
+    // #8142, run 39113747960: a .py file, in scope only as a sibling test.
+    ["plugins/planning/surface/test_round.py", ["plugins/planning/surface/round.py"]],
+    // #8178, run 39117146913: the PR's kept thread paths.
+    [
+      "plugins/planning/README.md",
+      ["plugins/session-flow/skills/retro/SKILL.md", "plugins/planning/skills/draft-goal-condition/SKILL.md", "plugins/planning/.claude-plugin/plugin.json"],
+    ],
+  ];
+  for (const [path, threads] of trips) {
+    assert.equal(inThreadScope(path, new Set(threads)), true, path);
+  }
+});
+
+test("hooks, manifests, agent rules and non-test code in the same plugin are refused", () => {
+  const named = new Set(["plugins/p/skills/s/SKILL.md"]);
+  for (const path of [
+    "plugins/p/hooks/hooks.json",
+    "plugins/p/hooks/guard.md",
+    "plugins/p/.claude-plugin/plugin.json",
+    "plugins/p/scripts/run.sh",
+    "plugins/p/skills/s/scripts/helper.py",
+    "plugins/p/lib/x.mjs",
+    "plugins/p/lib/x.js",
+    "plugins/p/lib/x.ts",
+    "plugins/p/lib/X.SH",
+    "plugins/p/AGENTS.md",
+    "plugins/p/skills/s/CLAUDE.md",
+    "plugins/p/.claude/rules/r.md",
+    "plugins/p/.claude/settings.json",
+  ]) {
+    assert.equal(inThreadScope(path, named), false, path);
+  }
+});
+
+test("a thread's own path stays in scope whatever its type, and a sibling test of code stays in scope", () => {
+  for (const path of ["plugins/p/hooks/guard.sh", "plugins/p/.claude-plugin/plugin.json", "plugins/p/AGENTS.md"]) {
+    assert.equal(inThreadScope(path, new Set([path])), true, path);
+  }
+  assert.equal(inThreadScope("plugins/p/hooks/guard.test.mjs", new Set(["plugins/p/hooks/guard.mjs"])), true);
+  assert.equal(inThreadScope("plugins/p/scripts/test_run.py", new Set(["plugins/p/README.md"])), false);
+});
+
+test("another plugin's file and an unnamed file outside plugins/ stay out of scope", () => {
+  assert.equal(inThreadScope("plugins/q/evals/c/samples/y.json", kept), false);
+  assert.equal(inThreadScope("plugins/pp/README.md", kept), false);
+  assert.equal(inThreadScope("src/b.js", kept), false);
+  assert.equal(inThreadScope("plugins/README.md", kept), false);
+});
+
+test("with no kept thread under the plugin, its files are refused", () => {
+  assert.equal(inThreadScope("plugins/p/evals/c/samples/y.json", new Set(["src/a.js"])), false);
+});
+
+// Names from the coordinator's addition; plugins/planning/surface/test_round.py
+// is the file #8142's run 39113747960 refused.
+test("a test file in the same directory as a kept thread's path is in scope", () => {
+  const surface = new Set(["plugins/planning/surface/round.py", "src/a.js", "README.md"]);
+  for (const path of [
+    "plugins/planning/surface/test_round.py",
+    "plugins/planning/surface/round_test.py",
+    "src/a.test.js",
+    "src/a.spec.ts",
+    "test_root.py",
+  ]) {
+    assert.equal(inThreadScope(path, surface), true, path);
+  }
+});
+
+test("outside plugins/, a test file in another directory, or a same-directory file that is no test, stays out of scope", () => {
+  const surface = new Set(["src/a.js"]);
+  for (const path of ["src/lib/a.test.js", "src/helper.js", "src/testing.js", "src/test_a.sh", "test_root.py"]) {
+    assert.equal(inThreadScope(path, surface), false, path);
+  }
+});
+
+test("a test file under .github/ is refused even beside a kept thread's path there", () => {
+  assert.equal(inThreadScope(".github/scripts/x.test.mjs", new Set([".github/scripts/x.mjs"])), false);
+});
+
+test("a path with an empty, . or .. segment, or a leading /, is refused before any other rule", () => {
+  for (const path of [
+    "plugins/p/evals/c/../../../../.github/workflows/x.yml",
+    "plugins/p/evals/c/./samples/y.json",
+    "plugins/p/evals/c//y.json",
+    "/src/a.js",
+    "./src/a.js",
+  ]) {
+    assert.equal(inThreadScope(path, new Set([...kept, path])), false, path);
+  }
+});
+
+test(".github/ is refused even when a kept thread names a file there", () => {
+  const github = new Set([".github/workflows/x.yml", ".GitHub/evals/c/a.md"]);
+  assert.equal(inThreadScope(".github/workflows/x.yml", github), false);
+  assert.equal(inThreadScope(".GitHub/evals/c/b.md", github), false);
+  assert.equal(isProtectedPath(".github"), true);
+  assert.equal(isProtectedPath(".githubx/a"), false);
+});