Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
| 2026-08-18 | claude/rag-eval-scope-detection-23b6a2 | e077c75cf46dddefcc21c5cbfeff4bbb2078fd48 | scripts/ci-change-scope.mjs rag_eval_changed classification (#SDQSFD) + change-scope self-test | Fixed: src/lib/rag/** now sets rag_eval_changed=true so eval:rag:offline and eval:rag:adversarial:offline are selected in CI and verify:pr-local; legacy flat patterns unchanged; four new self-test assertions incl. the first negative case; no workflow edit needed (ci.yml and verify-pr-local.mjs consume the classifier output). PR #2127. | npm run verify:pr-local (18/18 steps, 671 test files / 7250 tests passed, exit 0); npm run check:ci-scope (self-test passed); verify:pr-local --dry-run --files src/lib/rag/rag-claim-support.ts before/after (check:rag:fixtures only -> eval:rag:offline + eval:rag:adversarial:offline); mutation check (removing the new pattern fails the self-test). No provider-backed command run. |
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
{
"version": 2,
"id": "35826a8b-8ebf-4b19-8319-7ee229d028ca",
"createdOn": "2026-08-18",
"action": "done",
"payload": {
"id": "#SDQSFD",
"outcome": "Fixed in this PR: scripts/ci-change-scope.mjs ragEvalPatterns now carries the directory prefix /^src\\/lib\\/rag\\//, so a PR touching only the post-#994 src/lib/rag/** subtree sets rag_eval_changed=true and selects eval:rag:offline plus eval:rag:adversarial:offline in both verify:pr-local and the CI safety/RAG eval job. Legacy src/lib/rag.ts and src/lib/rag-*.ts patterns kept unchanged. Change-scope self-test extended with three positive cases (src/lib/rag/rag.ts, src/lib/rag/answer-composition.ts, src/lib/rag/rag-claim-support.ts) and the file's first negative rag_eval_changed assertion (src/lib/app-modes.ts stays false). No workflow edit needed: ci.yml and verify-pr-local.mjs both consume the classifier output rather than duplicating the regex. Note: the row proposed src/lib/answer-follow-up.ts as the negative case, but it is already rag_eval_changed=true via the pre-existing answer(-*)?.ts alternation and was deliberately left that way.",
"baseRowFingerprint": "5e7063d22140d3557fcdc986fbd112aa9a678485263a9a4ea9b53ba5598e43e7"
}
}
31 changes: 31 additions & 0 deletions scripts/ci-change-scope.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,14 @@ const ragEvalPatterns = [
"scripts/fixtures",
"src/app/api/answer",
"src/app/api/search",
// #SDQSFD: the RAG stack moved into `src/lib/rag/` in #994, but the flat
// `src/lib/rag-*.ts` regex below was never widened, so a PR touching only
// `src/lib/rag/rag.ts` classified as rag_eval_changed=false and skipped both
// `eval:rag:offline` and `eval:rag:adversarial:offline` in CI and
// verify:pr-local. PR #2065 reached main that way; the live canary, not the
// offline harness, caught it. Directory prefix (not `^src/lib/rag`) so the
// whole extracted subtree is covered whatever a file is later named.
/^src\/lib\/rag\//,
/^src\/lib\/(?:rag(?:-[^/]+)?|smart-rag-api|clinical-search|clinical-query-mode|retrieval(?:-[^/]+)?|answer(?:-[^/]+)?|citations|cross-document-synthesis|evidence(?:-[^/]+)?|ranking-config|source(?:-[^/]+)?|chunking|document-index-units|query-privacy|owner-scope|corpus-grounding|indexed-source-formatting)\.ts$/,
/^src\/components\/(?:.*\/)?(?:answer|source|citation)[^/]*\.tsx?$/i,
/^scripts\/(?:check-rag-fixtures|check-rag-adversarial-fixtures|rag-adversarial-contract|test-rag-offline)\.mjs$/,
Expand Down Expand Up @@ -1038,6 +1046,29 @@ function selfTest() {
rag_eval_changed: true,
source_changed: true,
});
// #SDQSFD: the extracted `src/lib/rag/` subtree (#994). Each of these alone
// must select the offline RAG contracts — `rag.ts` is the orchestrator and
// `answer-composition.ts` carries no `rag-` filename prefix at all, so the
// flat `src/lib/rag-*.ts` regex missed it entirely.
assertScope("rag-directory-orchestrator", ["src/lib/rag/rag.ts"], {
rag_eval_changed: true,
source_changed: true,
});
assertScope("rag-directory-answer-composition", ["src/lib/rag/answer-composition.ts"], {
rag_eval_changed: true,
source_changed: true,
});
assertScope("rag-directory-claim-support", ["src/lib/rag/rag-claim-support.ts"], {
rag_eval_changed: true,
source_changed: true,
});
// The other direction: the directory prefix must stay a directory prefix. An
// unrelated `src/lib` module is still an executable change, but it must not
// drag the offline RAG contracts onto every PR.
assertScope("non-rag-lib-module-skips-rag-eval", ["src/lib/app-modes.ts"], {
rag_eval_changed: false,
source_changed: true,
});
assertScope("rag-fixture-checker", ["scripts/check-rag-fixtures.mjs"], {
rag_eval_changed: true,
source_changed: true,
Expand Down
Loading