From 8725137530abc267d78d3ea915fc5df9852f6a50 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Sun, 4 Oct 2026 22:20:35 -0400 Subject: [PATCH] fix(discovery): exempt flagged first-party content claims from the single-publisher cap #6389 made the single-source content-claim exception vendor-neutral but left the single-publisher rule unchanged, so a vendor's own docs page (pool = vendor = apparent subject) still read as a single-publisher fact needing subject_pool and capped at MEDIUM. A flagged content claim's subject is the artifact: it carries no subject_pool, the cap does not apply, and a claim carries single_source: or subject_pool:, never both. The exception also covers a publisher's own help text. Verifier, sidecar schema, gotchas, evals and contract test follow. Salvaged from #6352. Co-Authored-By: Claude Opus 5.5 --- plugins/discovery/.claude-plugin/plugin.json | 2 +- plugins/discovery/CHANGELOG.md | 6 ++++++ plugins/discovery/agents/research-verifier.md | 5 ++++- plugins/discovery/scripts/contract.test.sh | 15 ++++++++++++--- .../skills/research/context/artifact-shape.md | 3 ++- .../skills/research/context/discipline.md | 6 +++--- .../discovery/skills/research/context/gotchas.md | 2 +- .../discovery/skills/research/evals/evals.json | 16 +++++++++++++++- 8 files changed, 44 insertions(+), 11 deletions(-) diff --git a/plugins/discovery/.claude-plugin/plugin.json b/plugins/discovery/.claude-plugin/plugin.json index 152a48feff..412cbd0822 100644 --- a/plugins/discovery/.claude-plugin/plugin.json +++ b/plugins/discovery/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "discovery", - "version": "0.28.9", + "version": "0.28.10", "description": "Discovery before changes: explore the local codebase, run multi-source external research, and reconstruct why a past decision was made from evidence outside the code. Each dispatches a subagent by default so the reading stays out of the main thread, with source tiers, falsification, recency gates and a coverage ledger, and persists EXPLORE.md / RESEARCH.md / INTENT.md handoff artifacts. A research sweep workflow (/discovery:research-sweep) backs deep research with adversarial claim verification.", "author": { "name": "Melodic Software", diff --git a/plugins/discovery/CHANGELOG.md b/plugins/discovery/CHANGELOG.md index 32e3d62f5a..546fd35e6a 100644 --- a/plugins/discovery/CHANGELOG.md +++ b/plugins/discovery/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog: discovery plugin +## [0.28.10] - 2026-10-04 + +### Fixed + +- `/discovery:research`: a flagged first-party content claim carries no `subject_pool`, and the single-publisher MEDIUM cap does not apply to it, since its subject is the artifact, not the publisher; a claim carries `single_source:` or `subject_pool:`, never both. The exception also covers a publisher's own help text. Verifiers had failed rows 4 and 7 on non-Anthropic first-party content claims whose quotes held on re-fetch. + ## [0.28.9] - 2026-10-04 ### Fixed diff --git a/plugins/discovery/agents/research-verifier.md b/plugins/discovery/agents/research-verifier.md index bc4bf5b208..1c3f2e4db7 100644 --- a/plugins/discovery/agents/research-verifier.md +++ b/plugins/discovery/agents/research-verifier.md @@ -84,7 +84,10 @@ claim carrying `subject_pool` is a single-publisher claim. Grade it against the [`${CLAUDE_PLUGIN_ROOT}/skills/research/context/discipline.md`](${CLAUDE_PLUGIN_ROOT}/skills/research/context/discipline.md): its `subject_pool` equals the one `pool` its Tier 0/1 sources share, it is worded as an attribution, it is at most MEDIUM, and it is not accepted. A claim whose Tier 0/1 sources all share -the `pool` of the claim's own subject but that carries no `subject_pool` fails row 4. +the `pool` of the claim's own subject but that carries no `subject_pool` fails row 4. A flagged +first-party content claim is the exception: its subject is the artifact, not the publisher, so it +carries `single_source:` and no `subject_pool`, and the single-publisher cap does not apply to it, +whoever the publisher is. A claim carrying both keys fails row 4. Fetch each page once, and read each file once; the rule is stated once in [`${CLAUDE_PLUGIN_ROOT}/reference/parent-contract.md`](${CLAUDE_PLUGIN_ROOT}/reference/parent-contract.md) diff --git a/plugins/discovery/scripts/contract.test.sh b/plugins/discovery/scripts/contract.test.sh index 0718c9f038..1cfcb13118 100755 --- a/plugins/discovery/scripts/contract.test.sh +++ b/plugins/discovery/scripts/contract.test.sh @@ -1051,8 +1051,9 @@ done # --------------------------------------------------------------------------- # 19. A first-party content claim with one possible publisher passes flagged # -# A claim about what a publisher's own page, file or changelog says has no -# second publisher to find. It passes rows 4 and 7 at `HIGH (single source)` +# A claim about what a publisher's own page, file, changelog or help text says, +# whoever the publisher is, has no second publisher to find. Its subject is the +# artifact, so the single-publisher cap does not apply to it. It passes rows 4 and 7 at `HIGH (single source)` # when it states why only one publisher exists; a repost is never a second # source, and a behavior claim still needs two corroborators. The flag travels # with the claim into the answer and into any record an edit rests on. @@ -1070,6 +1071,14 @@ assert_present 'discipline 5 points at the single-source exception' \ 'skills/research/SKILL.md' "^5\. .*\"Single-source first-party content claims\"" assert_present 'discipline.md carries the single-source section' \ 'skills/research/context/discipline.md' "^$single_heading$" +assert_absent 'no file scopes first-party content claims to one publisher' \ + 'what a named Anthropic (page|file)' +assert_present 'a content claim carries no subject_pool and escapes the cap' \ + 'skills/research/context/discipline.md' '^\*\*A content claim is not a single-publisher fact\.\*\*.*carries no `subject_pool` and the cap' +assert_present 'the single-publisher definition excludes flagged content claims' \ + 'skills/research/context/discipline.md' '^A claim is single-publisher when.*never fits this definition.*`single_source:` or `subject_pool:`, never both' +assert_present 'the verifier exempts a flagged content claim from the cap' \ + 'agents/research-verifier.md' 'single-publisher cap does not apply to it' assert_present 'discipline.md defines the HIGH (single source) level' \ 'skills/research/context/discipline.md' '^- \*\*HIGH \(single source\)\*\*' assert_present 'discipline.md lets a flagged claim ground a code edit with the flag carried' \ @@ -1092,7 +1101,7 @@ assert_present 'gotchas name the repost trap' \ 'skills/research/context/gotchas.md' '^- \*\*Counting a repost as the second source\.\*\*' assert_present 'research-deep keeps the flag through the synthesis' \ 'skills/research-deep/SKILL.md' 'keeps its `single source` flag' -for name in single-source-content-claim-passes-flagged behavior-claim-from-one-page-is-a-gap; do +for name in single-source-content-claim-passes-flagged non-anthropic-first-party-content-claim-passes-flagged behavior-claim-from-one-page-is-a-gap; do assert_present "research evals grade $name" 'skills/research/evals/evals.json' "\"name\": \"$name\"" done single_owners="$(surface | xargs grep -lE -- "^$single_heading$" 2>/dev/null | wc -l | tr -d ' ')" diff --git a/plugins/discovery/skills/research/context/artifact-shape.md b/plugins/discovery/skills/research/context/artifact-shape.md index cba2e785be..9ec16da74c 100644 --- a/plugins/discovery/skills/research/context/artifact-shape.md +++ b/plugins/discovery/skills/research/context/artifact-shape.md @@ -120,7 +120,8 @@ corroborator. **`subject_pool:` marks a single-publisher claim**, one whose every Tier 0/1 source is the publisher speaking about itself. It names that publisher and equals the one `pool` those sources share, so the verifier grades the label off the header, beside `pool`, for criterion 4. Omit the key -on every other claim. Rule: `discipline.md`'s "Single-publisher facts". +on every other claim, including a `HIGH (single source)` content claim, whose subject is the +artifact. Rule: `discipline.md`'s "Single-publisher facts". **`measures:`, `inference:`, and `qualifiers:` make criterion 12 gradeable off the artifact**, as `sources[]` does for criterion 4: a URL and a pool cannot show whether a source measured the claim's diff --git a/plugins/discovery/skills/research/context/discipline.md b/plugins/discovery/skills/research/context/discipline.md index e608224bf4..46f9673482 100644 --- a/plugins/discovery/skills/research/context/discipline.md +++ b/plugins/discovery/skills/research/context/discipline.md @@ -49,7 +49,7 @@ Every accepted claim has at least one Tier 0/1 source plus two independent corro ## Single-source first-party content claims -Some claims can have only one publisher. A **first-party content claim** states what a vendor's or maintainer's own official page, file or changelog says about its own product. The content exists in one place, so searching for an independent second source finds only copies of it. Such a claim passes criterion 4 with no counted corroborator when all of these hold: +Some claims can have only one publisher. A **first-party content claim** states what a vendor's or maintainer's own official page, file, changelog or help text says about its own product. The content exists in one place, so searching for an independent second source finds only copies of it. Such a claim passes criterion 4 with no counted corroborator when all of these hold: - **The claim states why only one publisher exists**, in the sidecar header's `single_source:` field. The verifier grades that reason; one that does not hold fails criterion 4 like any other uncorroborated claim. - **A repost is not a second source.** A blog post, forum answer, mirror or synthesis answer that quotes or restates the page shares its pool. Record it with the page's `pool`, never count it, and keep the flag: a repost does not turn the claim into a corroborated one. @@ -59,7 +59,7 @@ The claim's confidence is `HIGH (single source)` (see "Confidence calibration"), **A behavior claim is not a content claim.** What a product does when it runs (a default that takes effect, a limit it enforces, an error it raises) can be checked by a live probe or found in an issue report, so it still needs two independent corroborators, whatever its docs page says. A content claim about a page and a behavior claim drawn from that page are two claims: split them, and only the content claim can carry the flag. -**A content claim is not a single-publisher fact.** Its subject is the named artifact, not the publisher. When the page states the publisher's own pricing, roadmap or internal metrics, "the page says X" can carry the flag, while X as a fact about the publisher is a separate claim under "Single-publisher facts" below and is never accepted. +**A content claim is not a single-publisher fact.** Its subject is the named artifact, not the publisher, so it carries no `subject_pool` and the cap in "Single-publisher facts" below does not apply to it, even though its one source is the publisher's. When the artifact states the publisher's own pricing, roadmap, internal metrics, or how its product behaves or performs, "the artifact says X" can carry the flag, while X as a fact about the publisher or its product is a separate claim under "Single-publisher facts" and is never accepted on the publisher's word alone. ## Recency gate (for libraries, tools, CLIs, APIs) @@ -322,7 +322,7 @@ Only HIGH and HIGH (single source) claims are accepted (the outcome gate enforce Some facts only their publisher can state: its own pricing, roadmap, internal metrics, or how its own product behaves. Two sources sharing a `pool` are one corroborator (`artifact-shape.md`), so the publisher's own pages count once toward criterion 4 however many repeat the fact, and such a fact never reaches criterion 7 or counts as independently corroborated. -A claim is single-publisher when every Tier 0/1 source behind it shares one `pool` and that pool is the claim's subject: the publisher speaking about itself. Carry it this way: +A claim is single-publisher when every Tier 0/1 source behind it shares one `pool` and that pool is the claim's subject: the publisher speaking about itself. A flagged first-party content claim never fits this definition: its subject is the artifact, and it is graded under "Single-source first-party content claims" above. A claim carries `single_source:` or `subject_pool:`, never both. Carry a single-publisher claim this way: - **Worded as an attribution.** The claim reads " states ...", never as a bare fact, in the sidecar, the evidence table and the answer. - **`subject_pool:` on the claim** names the subject publisher and equals the one `pool` its Tier 0/1 sources share (schema: `artifact-shape.md`). A claim that fits the definition and carries no `subject_pool` is unlabeled, and the verifier fails it on criterion 4. diff --git a/plugins/discovery/skills/research/context/gotchas.md b/plugins/discovery/skills/research/context/gotchas.md index 4a04a4f609..f3c029c08a 100644 --- a/plugins/discovery/skills/research/context/gotchas.md +++ b/plugins/discovery/skills/research/context/gotchas.md @@ -54,7 +54,7 @@ outcome gate's artifact-grounded criteria, or not at all. scenario checks ask whether the source covers the claim's product line and situation. A `historical` source is labeled and never counted. - **Counting a repost as the second source.** A blog post or synthesis answer that restates a - publisher's own page is that page again. Counting it lets a single-publisher claim pass criterion 4 as + publisher's own page is that page again. Counting it lets a first-party content claim pass criterion 4 as corroborated, and the `single source` flag that should travel with the claim disappears. Record the repost under the page's pool and flag the claim. The opposite slip costs as much: flagging a behavior claim because its docs page is the only one found, when a probe or an issue could diff --git a/plugins/discovery/skills/research/evals/evals.json b/plugins/discovery/skills/research/evals/evals.json index 058732d5e7..b32515dde1 100644 --- a/plugins/discovery/skills/research/evals/evals.json +++ b/plugins/discovery/skills/research/evals/evals.json @@ -421,7 +421,7 @@ "name": "single-source-content-claim-passes-flagged", "prompt": "Research what Anthropic's Claude Code changelog says about one named release. [The dispatched discovery:researcher returns a well-formed payload with verification: pending, and both gates exit 0. One accepted claim reads 'the changelog entry for that release lists the change'. Its sidecar header records confidence: HIGH (single source) and single_source: 'only Anthropic publishes this changelog; every copy found elsewhere quotes it'. Its sources are the changelog file, fetched this turn, as role: primary, plus a blog post and a newsletter issue, each quoting that changelog entry and each recorded under the changelog's pool.]", "files": [], - "expected_output": "Briefs the sibling verifier on rows 4, 7, 12 and 14. The verifier grades row 4 on its single-source branch: the claim states what a named Anthropic file says, the stated reason holds, and the blog post and newsletter are reposts recorded under the changelog's pool and not counted as corroborators. Row 4 passes without two corroborators, and row 7 passes at HIGH (single source). The parent writes verification: pass and presents the claim with its single source flag visible in the evidence table and the answer. A code edit that later rests on the claim carries the flag in the record beside it.", + "expected_output": "Briefs the sibling verifier on rows 4, 7, 12 and 14. The verifier grades row 4 on its single-source branch: the claim states what a named first-party file says, the stated reason holds, and the blog post and newsletter are reposts recorded under the changelog's pool and not counted as corroborators. Row 4 passes without two corroborators, and row 7 passes at HIGH (single source). The parent writes verification: pass and presents the claim with its single source flag visible in the evidence table and the answer. A code edit that later rests on the claim carries the flag in the record beside it.", "expectations": [ "The verifier is briefed on rows 4, 7, 12 and 14 by number", "Row 4 is graded on the claim's single_source: reason, not failed for lacking two independent corroborators", @@ -485,6 +485,20 @@ "The fixed artifact records accepted: 0 and its Summary opens with 'Inconclusive: no claim accepted.' and names the blocking Conflicts", "The result is presented as inconclusive, not as an answer and not as a gate failure once the count and the line are in place" ] + }, + { + "id": 34, + "name": "non-anthropic-first-party-content-claim-passes-flagged", + "prompt": "Research what ExampleCorp's docs for its ExampleBrowser CLI say about the video-recording flag. [The dispatched discovery:researcher returns a well-formed payload with verification: pending, and all three gates exit 0. One accepted claim reads 'the ExampleBrowser CLI reference page states that --record-video writes a WebM file per session'. Its sidecar header records confidence: HIGH (single source), single_source: 'ExampleCorp publishes the ExampleBrowser CLI and is the only publisher of its reference page; copies found elsewhere quote it', and no subject_pool. Its only Tier 0/1 source is that reference page, fetched this turn, as role: primary, under pool ExampleCorp. A tutorial blog quoting the page is recorded under the same pool.]", + "files": [], + "expected_output": "Briefs the sibling verifier on rows 4, 7, 12 and 14. The verifier grades row 4 on the single-source branch even though the publisher is not Anthropic: the claim states what ExampleCorp's own reference page says about ExampleCorp's own CLI, the reason holds, and the tutorial is a repost under the page's pool. The claim's subject is the page, not ExampleCorp, so it needs no subject_pool and the single-publisher MEDIUM cap does not apply. Rows 4 and 7 pass at HIGH (single source). Whether the CLI actually writes one WebM per session is a separate behavior claim that this flag does not cover.", + "expectations": [ + "Row 4 is graded on the claim's single_source: reason, not failed because the publisher is not Anthropic", + "The claim is not failed for lacking subject_pool, and is not capped at MEDIUM as a single-publisher fact", + "The tutorial blog is treated as a repost: recorded under the page's pool, not counted", + "Row 7 passes with the claim at HIGH (single source)", + "The runtime behavior of the flag is named as a separate claim the single source flag does not cover" + ] } ] }