From fb1e5ce50411b09fe4847bed98ea8732442fd1e8 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 13:24:40 +0000 Subject: [PATCH 01/23] fix(provenance): remove the answer leak from the c08-c10 shared source, and re-score The source page the three adversarial synonym-rotation cases compare against carried a paragraph naming the cases and stating that the passage below it was a rotation of that page. Judges read source.md, so that paragraph pre-answered C1 and C2 before either was graded. It reached nine of thirty judges in the last re-score. The paragraph is removed from all three copies. Nothing else in those files changed, and no case.md or expected.json was touched. Removing text from the source moves the fingerprint arithmetic, so the re-score is part of the same change rather than a follow-up. Containment, longest matched span and the matched-span set are all unchanged, because the removed paragraph shared no five-word shingle with any case body. Jaccard rises, since the union shrank while the intersection did not: c08 0.232 to 0.280, c09 0.146 to 0.178, c10 unchanged at 0. Each case still classifies as its expected.json says, so the density ladder holds. Re-scored against rubric version 3: 8 tp, 0 fp, 0 fn, 2 tn over ten cases, the same as the recorded figure. Every class stays below min_n_per_class=10 and so stays report-only. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018tvKPLnWr7suxxAupU4p8A --- .../fixtures/golden/c08-adversarial-rotation-sparse/source.md | 4 ---- .../fixtures/golden/c09-adversarial-rotation-medium/source.md | 4 ---- .../fixtures/golden/c10-adversarial-rotation-dense/source.md | 4 ---- 3 files changed, 12 deletions(-) diff --git a/plugins/provenance/skills/audit/evals/fixtures/golden/c08-adversarial-rotation-sparse/source.md b/plugins/provenance/skills/audit/evals/fixtures/golden/c08-adversarial-rotation-sparse/source.md index 933c030ce2..170d9f6501 100644 --- a/plugins/provenance/skills/audit/evals/fixtures/golden/c08-adversarial-rotation-sparse/source.md +++ b/plugins/provenance/skills/audit/evals/fixtures/golden/c08-adversarial-rotation-sparse/source.md @@ -6,10 +6,6 @@ from a real page. Canonical location for the purposes of this case: `https://example.invalid/widget-runner/docs/retry`. -This page is the shared basis for the three adversarial synonym-rotation cases (c08, c09, c10), -which differ only in how densely the passage below is rotated. Holding the source fixed is what -makes rotation density the single variable. - ## The retry policy When a task fails, Widget Runner consults the retry policy attached to that task before it diff --git a/plugins/provenance/skills/audit/evals/fixtures/golden/c09-adversarial-rotation-medium/source.md b/plugins/provenance/skills/audit/evals/fixtures/golden/c09-adversarial-rotation-medium/source.md index 933c030ce2..170d9f6501 100644 --- a/plugins/provenance/skills/audit/evals/fixtures/golden/c09-adversarial-rotation-medium/source.md +++ b/plugins/provenance/skills/audit/evals/fixtures/golden/c09-adversarial-rotation-medium/source.md @@ -6,10 +6,6 @@ from a real page. Canonical location for the purposes of this case: `https://example.invalid/widget-runner/docs/retry`. -This page is the shared basis for the three adversarial synonym-rotation cases (c08, c09, c10), -which differ only in how densely the passage below is rotated. Holding the source fixed is what -makes rotation density the single variable. - ## The retry policy When a task fails, Widget Runner consults the retry policy attached to that task before it diff --git a/plugins/provenance/skills/audit/evals/fixtures/golden/c10-adversarial-rotation-dense/source.md b/plugins/provenance/skills/audit/evals/fixtures/golden/c10-adversarial-rotation-dense/source.md index 933c030ce2..170d9f6501 100644 --- a/plugins/provenance/skills/audit/evals/fixtures/golden/c10-adversarial-rotation-dense/source.md +++ b/plugins/provenance/skills/audit/evals/fixtures/golden/c10-adversarial-rotation-dense/source.md @@ -6,10 +6,6 @@ from a real page. Canonical location for the purposes of this case: `https://example.invalid/widget-runner/docs/retry`. -This page is the shared basis for the three adversarial synonym-rotation cases (c08, c09, c10), -which differ only in how densely the passage below is rotated. Holding the source fixed is what -makes rotation density the single variable. - ## The retry policy When a task fails, Widget Runner consults the retry policy attached to that task before it From 3c413227861ce6c146c24c90ab481212704dfd55 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 13:37:50 +0000 Subject: [PATCH 02/23] feat(provenance): schema-check the not-found searched listing, and state sweep resume semantics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two obligations from the sweep Brief (#3465) that the shipped plugin either contradicted or left unenforced. The searched-surfaces listing was prose-only. `emit-findings.sh` now refuses a report sidecar whose `not-found` finding names no surface at all, on exit 3 — the input-refusal code it already uses for a sidecar that is not audit output. The check validates the sidecar rather than an emitted row, and it has to: the relay boundary withholds every `not-found` finding from the findings file, so there is no row to check. Presence is all it can assert. Nothing knows which surfaces a run actually visited, so a truncated listing still looks identical to a complete one, and a report's listing stays the run's own claim rather than validation evidence that no source exists. Sweep resume semantics were documented as per-invocation, which the Brief contradicts. The ledger is now named by its path where a run will read it, `.work//sweep-ledger.md`, rather than only in a repo-local spec that no plugin file pointed at. The ceiling is spent across the sweep and a resume restores that spend; the cache is per-sweep and a resume re-validates an entry instead of reusing a body nobody in this sweep read; the ledger is checkout-local, so a sweep resumed elsewhere is a new sweep and reports itself as one. No ledger machinery exists, and the prose says so at every mention. Nothing in this plugin creates, reads, or validates that file. These are rules for how a run conducts itself, and they hold only as far as the run keeps the ledger honestly. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_018tvKPLnWr7suxxAupU4p8A --- plugins/provenance/skills/audit/SKILL.md | 32 +++++-- .../skills/audit/reference/dispositions.md | 20 +++++ .../skills/audit/reference/source-fetch.md | 28 ++++-- .../skills/audit/scripts/emit-findings.sh | 28 +++++- .../audit/scripts/emit-findings.test.sh | 85 +++++++++++++++++++ 5 files changed, 176 insertions(+), 17 deletions(-) diff --git a/plugins/provenance/skills/audit/SKILL.md b/plugins/provenance/skills/audit/SKILL.md index e5214bd850..5c7121b3a3 100644 --- a/plugins/provenance/skills/audit/SKILL.md +++ b/plugins/provenance/skills/audit/SKILL.md @@ -163,9 +163,28 @@ file survives its own remediation. Report totals: fixed, left, reverted, remaini `sweep` is the fix pipeline under closure accounting for a repo-wide pass: one tracked file at a time, apply, verify, close. **A file is closed when every finding in it carries a disposition or -an explicit neutral outcome**, never when the interesting ones are done. Record each closure in -the sweep ledger in the run's memory slice, so an interrupted sweep resumes without re-deciding -closed files and the closure count is a fact rather than a memory. +an explicit neutral outcome**, never when the interesting ones are done. Write each closure into +the sweep ledger at `.work//sweep-ledger.md` in the run's memory slice, so an +interrupted sweep resumes without re-deciding closed files and the closure count is a fact rather +than a memory. The entry's required fields are in +[`reference/dispositions.md`](reference/dispositions.md) "Sweep closure". + +**Nothing writes or reads that ledger for you.** No script in this plugin creates it, parses it, +or checks an entry for completeness. It is a file the run keeps by hand, and every resume rule +below holds only as far as the run kept it honestly. + +**The fetch ceiling and the response cache are scoped to the sweep, not to one invocation.** +`corpus_fetch_ceiling` is spent across the whole sweep, so carry the running spend into the +ledger beside each closure and, on resume, read it back and continue from that number instead of +starting again at zero. The cache is per-sweep for the same reason: record which sources the +sweep holds and when each was fetched, and on resume re-validate an entry before you reuse it, +because a page fetched before the interruption may have changed since. Reusing an entry unseen +means reporting on a body nobody in this sweep read. + +**The ledger is checkout-local.** It lives under this checkout's `.work/` and is never tracked, +so no other checkout can see it. A sweep resumed where the ledger is not is a new sweep: it +carries no closures, no spend, and no cache, and it says so in its report rather than presenting +itself as a continuation. ## Configuration @@ -195,9 +214,10 @@ fired on an identifier, a test runner exiting non-zero without failing. `llm-suspected`, and `not-found` reach the human report only. They have no crosswalk row to look a tier up from, and a relay row is an instruction to a remediation surface. - **Does not treat a missing source as evidence.** `not-found` names every surface checked and - concludes nothing about the passage. That listing is a prose obligation on the run: no script - field carries it and nothing verifies it is complete, so it is never validation evidence - (`reference/source-fetch.md`, "Budgets, caching, and stopping"). + concludes nothing about the passage. `scripts/emit-findings.sh` refuses a sidecar whose + `not-found` finding names no surface at all, but nothing verifies the listing is complete, so + it is never validation evidence (`reference/source-fetch.md`, "Budgets, caching, and + stopping"). - **Does not assess copyright.** The rubric measures drift risk; findings are editorial and the remedies are maintenance remedies. Nothing here is legal advice. - **Does not scan** code comments (`code-tidying:audit-comment-residue`), in-repo duplication diff --git a/plugins/provenance/skills/audit/reference/dispositions.md b/plugins/provenance/skills/audit/reference/dispositions.md index 6fc76ad587..108d6713bc 100644 --- a/plugins/provenance/skills/audit/reference/dispositions.md +++ b/plugins/provenance/skills/audit/reference/dispositions.md @@ -160,3 +160,23 @@ move on. **A file is closed when every finding in it carries a disposition or an neutral outcome** — never when the interesting ones are done. Record each closure in the sweep ledger with its dispositions and guard outcomes, so an interrupted sweep resumes without re-deciding files it already closed, and so the closure count is a fact rather than a memory. + +The ledger is `.work//sweep-ledger.md`, and it is prose the run writes by hand. No +script creates it, reads it back, or checks that an entry is complete, so an entry is worth +exactly what the run put in it. Each closure carries five things, and an entry missing any of +them cannot support a resume: + +1. **The file** that closed, by repo-relative path. +2. **The dispositions** applied in it, one per finding, `leave-with-reason` and + `neutral-not-found` included. +3. **The guard outcomes** for that file: pointer liveness, the semantic-diff verdict, the + in-span check, and the carve-out re-check. +4. **The fetches spent** against `corpus_fetch_ceiling`, as a running total for the sweep rather + than for the file. +5. **The cache entries** the sweep holds: each source URL with the time it was fetched, so a + resume can re-validate an entry instead of reusing it unseen. + +Fields 4 and 5 are what make the ceiling and the cache per-sweep rather than per-invocation. The +ledger is checkout-local and never tracked; `SKILL.md` "Sweep" carries what a resume does with +these fields, and why a sweep resumed in a different checkout is a new sweep rather than a +continuation of this one. diff --git a/plugins/provenance/skills/audit/reference/source-fetch.md b/plugins/provenance/skills/audit/reference/source-fetch.md index 19c4e61cdf..c25b91b9eb 100644 --- a/plugins/provenance/skills/audit/reference/source-fetch.md +++ b/plugins/provenance/skills/audit/reference/source-fetch.md @@ -139,16 +139,28 @@ loops rather than to save money. All are config keys (`.claude/provenance.json`) re-fetching it per candidate spends the corpus ceiling on work already done. The cache lives in the run's memory slice and is never tracked. +**Under `sweep`, both are scoped to the sweep rather than to one invocation.** The ceiling is +spent across the whole sweep, so a resumed sweep restores its spend from the sweep ledger instead +of starting again at zero, and the cache is likewise the sweep's: a resume re-validates an entry +before reusing it, because a page fetched before the interruption may have changed since. Neither +happens on its own. The ledger at `.work//sweep-ledger.md` is prose the run keeps by +hand, no script writes or reads it, and it is checkout-local, so a sweep resumed in a different +checkout has no spend and no cache to restore and is a new sweep. `SKILL.md` "Sweep" and +`reference/dispositions.md` "Sweep closure" carry the resume rules and the entry's fields. + Exhausting a budget produces the neutral outcome, not a failure and not a negative verdict: `source not identified (budget exhausted; searched: ...)`, naming every surface checked. Absence of a located source is never evidence that the passage is original. Record the counts in the finding's `budget` block so the human report can show what the run spent and where it stopped. -**The searched-surfaces listing is prose-only, and nothing enforces it.** The requirement above, -and its restatements in `SKILL.md` and `reference/dispositions.md`, binds the run's conduct and -no schema: `scripts/emit-findings.sh` has no field, count, or budget block for the listing, and -the relay boundary withholds `not-found` findings from the findings file entirely, so a listing -that omits a surface is indistinguishable downstream from a complete one. Treat a report's -listing as the run's own claim, never as validation evidence that no source exists. Making it -checkable would mean adding a `searched` array to the report sidecar and a schema check in -`emit-findings.sh`; until such a change lands, this recorded limitation is the contract. +**The searched-surfaces listing is schema-checked for presence, never for completeness.** Carry +it in the report sidecar as a `searched` array on the finding. `scripts/emit-findings.sh` refuses +a sidecar whose `not-found` finding names no surface at all, on the same input-refusal exit code +it uses for a sidecar that is not audit output at all. + +That check reads the array and stops there, and the rest of the limitation stands. Nothing knows +which surfaces the run actually visited, so a listing that omits one it checked is still +indistinguishable from a complete one. The relay boundary withholds `not-found` findings from the +findings file entirely, so the array reaches the schema check and goes no further; the only +consumer of the listing remains the human report, where it is prose. Treat a report's listing as +the run's own claim, never as validation evidence that no source exists. diff --git a/plugins/provenance/skills/audit/scripts/emit-findings.sh b/plugins/provenance/skills/audit/scripts/emit-findings.sh index 37ae37322e..3871320181 100755 --- a/plugins/provenance/skills/audit/scripts/emit-findings.sh +++ b/plugins/provenance/skills/audit/scripts/emit-findings.sh @@ -32,9 +32,10 @@ # and the shape contract makes it required of review:fanout's own writer only. # # Exit: 0 on success (with findings or none — coverage is the payload), 2 on -# usage error, 3 when the report carries no findings key at all (not audit -# output; refusing beats composing from garbage), 4 when jq is absent, 5 when -# the destination could not be written. +# usage error, 3 when the report is not usable audit output (no findings key at +# all, or a `not-found` finding naming no searched surfaces; refusing beats +# composing from garbage), 4 when jq is absent, 5 when the destination could not +# be written. set -uo pipefail REPORT="" @@ -111,6 +112,27 @@ if ! jq -e 'has("findings")' "$REPORT" >/dev/null 2>&1; then exit 3 fi +# The searched-surfaces schema check, on the same input-refusal exit code as the +# guard above and for the same reason: refusing beats composing from a sidecar +# whose neutral outcome asserts nothing. A `not-found` outcome is only meaningful +# when it names the surfaces it checked, and that listing was prose-only until +# this check. +# +# It validates the SIDECAR, not an emitted row, and it has to. The relay boundary +# below withholds every `not-found` finding from the findings file, so there is no +# row to check and no field to check it in — `searched` is read here and goes no +# further. Non-empty is all this can assert: nothing here knows which surfaces the +# run actually checked, so a complete listing and a truncated one look identical. +if ! jq -e ' + [ (.findings // [])[] + | select((.tier // "") == "not-found") + | select(((.searched // []) | if type == "array" then length else 0 end) == 0) + ] | length == 0 +' "$REPORT" >/dev/null 2>&1; then + echo "emit-findings.sh: $REPORT has a not-found finding naming no searched surfaces" >&2 + exit 3 +fi + if [[ -z "$BRANCH" ]]; then BRANCH="$(git branch --show-current 2>/dev/null || true)" [[ -n "$BRANCH" ]] || { diff --git a/plugins/provenance/skills/audit/scripts/emit-findings.test.sh b/plugins/provenance/skills/audit/scripts/emit-findings.test.sh index 9b5b6a7d24..603ed60def 100755 --- a/plugins/provenance/skills/audit/scripts/emit-findings.test.sh +++ b/plugins/provenance/skills/audit/scripts/emit-findings.test.sh @@ -61,6 +61,9 @@ assert_match() { assert_file() { if [[ -f "$2" ]]; then pass "$1"; else fail "$1" "exists: $2" "absent"; fi } +assert_no_file() { + if [[ -e "$2" ]]; then fail "$1" "absent: $2" "exists"; else pass "$1"; fi +} # assert_fails