diff --git a/apps/loopover-ui/content/docs/fairness-methodology.mdx b/apps/loopover-ui/content/docs/fairness-methodology.mdx index 3b87a98f3c..ecc6ec2db8 100644 --- a/apps/loopover-ui/content/docs/fairness-methodology.mdx +++ b/apps/loopover-ui/content/docs/fairness-methodology.mdx @@ -16,8 +16,22 @@ Nothing here is a summary of intent. Every rule below is the rule the code imple **Want the machine-checkable version?** `npx -p @loopover/mcp loopover-verify` recomputes the - published commitments from the public endpoints and prints a PASS/FAIL table. It needs no - credentials. See [Verify this review](/docs/verify-this-review). + published commitments and prints a PASS/FAIL table. It needs no credentials. See + [Verify this review](/docs/verify-this-review). + + Two surfaces answer different questions, because they hold different data (#9940). Aggregate + stats and eval-score records come from `api.loopover.ai`, the default. The **anchored decision + ledger** lives on the Orb that actually reviews — its own hash chain, signed and anchored to + Rekor — so ledger and anchor checks need that one: + + ```bash + npx -p @loopover/mcp loopover-verify # stats + eval scores + npx -p @loopover/mcp loopover-verify --base-url https://shots.loopover.ai # ledger + anchors + ``` + + The public API's own ledger is empty by design: it aggregates outcomes reported by Orbs rather + than holding their chains. A verifier run against it reports the ledger checks as *skipped*, + not failed. ## The three data sources are not interchangeable diff --git a/apps/loopover-ui/content/docs/verify-this-review.mdx b/apps/loopover-ui/content/docs/verify-this-review.mdx index 06aa436c5c..86be9335f7 100644 --- a/apps/loopover-ui/content/docs/verify-this-review.mdx +++ b/apps/loopover-ui/content/docs/verify-this-review.mdx @@ -16,6 +16,19 @@ figure stops supporting a conclusion — see the [fairness methodology](/docs/fa For a single command that recomputes the published commitments and exits non-zero if any fail, run `npx -p @loopover/mcp loopover-verify`. +**Two surfaces, because they hold different things** (#9940). The commands on this page use +`api.loopover.ai`, which serves aggregate stats and eval-score records. The **anchored decision +ledger** — the hash chain, its signed checkpoints, and the Rekor anchors — lives on the Orb that +performs the reviews, at `https://shots.loopover.ai`. Point the ledger and anchor checks there: + +```bash +curl -s https://shots.loopover.ai/v1/public/decision-ledger/verify | jq '{ok, tipSeq, totalCount}' +npx -p @loopover/mcp loopover-verify --base-url https://shots.loopover.ai +``` + +The public API's own ledger is empty by design — it aggregates outcomes Orbs report, rather than +holding their chains — so a run against it reports the ledger checks as *skipped*, never failed. + Everything below runs read-only against a corpus export and pure functions from `@loopover/engine`. Nothing posts anywhere, and nothing needs a LoopOver API key. diff --git a/packages/loopover-mcp/bin/loopover-verify.ts b/packages/loopover-mcp/bin/loopover-verify.ts index b2549eb243..8f66c83c3e 100644 --- a/packages/loopover-mcp/bin/loopover-verify.ts +++ b/packages/loopover-mcp/bin/loopover-verify.ts @@ -109,11 +109,14 @@ export async function runVerify(args: readonly string[], baseUrlOverride?: strin // Fetched together: they are independent reads, and a verifier that serialises four round trips for no // reason is a slower verifier with no compensating property. - const [scoresOutcome, statsOutcome, checkpointOutcome, keysOutcome] = await Promise.all([ + const [scoresOutcome, statsOutcome, checkpointOutcome, keysOutcome, ledgerOutcome] = await Promise.all([ apiGet<{ records?: VerifiableEvalRecord[] }>(baseUrl, "/v1/public/eval-scores"), apiGet<{ totals?: { handled?: unknown }; reviewParity?: { verdicts?: unknown } }>(baseUrl, "/v1/public/stats"), apiGet<{ signed?: unknown; signingInput?: unknown }>(baseUrl, "/v1/public/decision-ledger/anchor-payload"), apiGet<{ keys?: unknown[] }>(baseUrl, "/v1/public/decision-ledger/anchor-key"), + // #9940: the ledger's own size, so an EMPTY surface is reported as such rather than as a missing + // signing key. Those are different findings and only one of them is a problem. + apiGet<{ totalCount?: unknown }>(baseUrl, "/v1/public/decision-ledger/verify"), ]); const records = scoresOutcome.ok && Array.isArray(scoresOutcome.value.records) ? scoresOutcome.value.records : []; @@ -141,7 +144,7 @@ export async function runVerify(args: readonly string[], baseUrlOverride?: strin // so an unavailable checkpoint is passed through as `undefined` and reported as a skip by the check // itself rather than being special-cased into a second, subtly different skip message here. const keys = keysOutcome.ok && Array.isArray(keysOutcome.value.keys) ? (keysOutcome.value.keys as Parameters[1]) : []; - results.push(await checkAnchorCheckpoint(checkpointOutcome.ok ? checkpointOutcome.value : undefined, keys)); + results.push(await checkAnchorCheckpoint(checkpointOutcome.ok ? checkpointOutcome.value : undefined, keys, ledgerOutcome.ok ? ledgerOutcome.value : undefined)); results.push( statsOutcome.ok diff --git a/packages/loopover-mcp/lib/verify-public-claims.ts b/packages/loopover-mcp/lib/verify-public-claims.ts index 1e12a7d73c..ddda3a5546 100644 --- a/packages/loopover-mcp/lib/verify-public-claims.ts +++ b/packages/loopover-mcp/lib/verify-public-claims.ts @@ -179,11 +179,31 @@ export async function checkCorpusCommitments( export async function checkAnchorCheckpoint( checkpoint: { signed?: unknown; signingInput?: unknown } | undefined, keys: readonly AnchorPublicKey[], + ledger?: { totalCount?: unknown } | undefined, ): Promise { const claim = "The current signed ledger checkpoint verifies offline against a published key"; const id = "anchor-checkpoint"; if (checkpoint === undefined || !isSignedAnchor(checkpoint.signed)) { - return { id, claim, status: "skip", detail: "no signed checkpoint published (anchor signing not configured, or the ledger is empty)" }; + // #9940: an EMPTY ledger and a misconfigured signer used to produce the same sentence, and the + // difference is the whole diagnosis. This surface holding no decisions is not a verifiability failure + // -- there is nothing to anchor -- whereas decisions with no signing key is a real gap. Conflating them + // sent me down the wrong path on a live deployment: I read "not configured" and went looking for a + // missing secret, when the deployment simply had no ledger and the anchoring worked fine elsewhere. + const totalCount = typeof ledger?.totalCount === "number" ? ledger.totalCount : null; + if (totalCount === 0) { + return { + id, + claim, + status: "skip", + detail: "this deployment's decision ledger is EMPTY (0 records), so there is nothing to anchor — check the deployment that actually records decisions, via --base-url", + }; + } + return { + id, + claim, + status: "skip", + detail: totalCount === null ? "no signed checkpoint published, and the ledger size is unknown" : `no signed checkpoint published, though the ledger holds ${totalCount} record(s) — anchor signing looks unconfigured here`, + }; } const signed = checkpoint.signed; if (keys.length === 0) { diff --git a/test/unit/verify-public-claims.test.ts b/test/unit/verify-public-claims.test.ts index 4525e86ebd..2d354bafaa 100644 --- a/test/unit/verify-public-claims.test.ts +++ b/test/unit/verify-public-claims.test.ts @@ -176,6 +176,25 @@ describe("checkAnchorCheckpoint", () => { expect(result.detail).toContain("not among"); }); + it("distinguishes an EMPTY ledger from a missing signing key (#9940)", async () => { + // These read identically before, and the difference is the whole diagnosis. Conflating them sent me + // looking for a missing secret on a deployment whose anchoring was working fine -- the surface simply + // held no ledger, because the records live on a different one. + const { checkAnchorCheckpoint } = await loadClaims(); + const empty = await checkAnchorCheckpoint(undefined, [], { totalCount: 0 }); + expect(empty.status).toBe("skip"); + expect(empty.detail).toContain("EMPTY"); + expect(empty.detail).toContain("--base-url"); + + // Records present but nothing signed: a real gap, and it must NOT read as "nothing to anchor". + const unsigned = await checkAnchorCheckpoint(undefined, [], { totalCount: 2190 }); + expect(unsigned.detail).toContain("2190"); + expect(unsigned.detail).not.toContain("EMPTY"); + + // Ledger size unknown (the endpoint itself was unreachable) is its own third case. + expect((await checkAnchorCheckpoint(undefined, [], undefined)).detail).toContain("ledger size is unknown"); + }); + it("skips when no checkpoint is published, and when no key is published to check one against", async () => { const { checkAnchorCheckpoint } = await loadClaims(); const { signed } = await realCheckpoint();