From 9382d79882bdfb9b1d0c2b7f45c3f2da2fac932d Mon Sep 17 00:00:00 2001 From: JSONbored <49853598+JSONbored@users.noreply.github.com> Date: Wed, 29 Jul 2026 07:24:50 -0700 Subject: [PATCH] fix(ledger): stop one unreconcilable row from hiding every row after it verifyDecisionLedger returned at the FIRST content_mismatch, so a single row whose preimage no longer matches became a denial-of-verification for the whole chain behind it. Found on a live instance: 83 rows (seq 5-257, 2026-07-26 to 07-27) left by the record-overwriting UPDATE that #9123 replaced with the revision scheme. Zero divergence in the 1,061 rows since -- the write path is fixed and the damage is historical -- but verification stopped at seq 5 and never examined the remaining 1,649 rows. Real tampering at seq 900 would have been invisible behind permanent damage at seq 5, on the one endpoint whose entire purpose is that a skeptic can trust it. Content mismatches now accumulate and the scan continues. That is safe precisely because the chain checks already passed for the row: sequence, predecessor and row_hash all reconciled, so prevHash for the next row is sound. A content mismatch is a statement about one row's preimage. A STRUCTURAL break is not -- a sequence gap or predecessor mismatch makes everything after it unverifiable -- so those still return immediately, and now actually get reached. The verdict is NOT softened: any mismatch is still ok:false, and the first is still reported as . What changes is that the scan finishes and reports contentMismatches, so "one bad row" and "eighty-three bad rows" stop looking identical, and a later structural break is no longer unreachable. Refs #9850 --- apps/loopover-ui/public/openapi.json | 2 +- src/openapi/spec.ts | 2 +- src/review/decision-record.ts | 41 ++++++++++++--- test/unit/decision-record.test.ts | 75 +++++++++++++++++++++++++++- 4 files changed, 109 insertions(+), 11 deletions(-) diff --git a/apps/loopover-ui/public/openapi.json b/apps/loopover-ui/public/openapi.json index 53ed127f68..cd014bafe2 100644 --- a/apps/loopover-ui/public/openapi.json +++ b/apps/loopover-ui/public/openapi.json @@ -20760,7 +20760,7 @@ "summary": "Verify a window of the hash-chained decision ledger (resumable via afterSeq)", "responses": { "200": { - "description": "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." + "description": "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, contentMismatches (#9850 -- the COUNT of rows whose preimage no longer matches the digest the chain committed to; a content mismatch no longer aborts the scan, because one unreconcilable row used to hide every row after it, so a structural break further along was unreachable. The verdict is unchanged: any mismatch is still ok:false and the first is still reported as `break`), and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." }, "409": { "description": "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | missing_record | content_mismatch | short_tail (a record newer than the verified tip has no chain entry — the truncated-tail signature) | unchained_record (an INTERIOR record has no chain entry — the failed-append signature). Records younger than the 5-minute append grace window are not reported: the record insert and its chain append are two writes moments apart, and a verify landing between them is not evidence of tampering." diff --git a/src/openapi/spec.ts b/src/openapi/spec.ts index 7dcb7c84fe..2c62a216ed 100644 --- a/src/openapi/spec.ts +++ b/src/openapi/spec.ts @@ -1882,7 +1882,7 @@ export function buildOpenApiSpec() { tags: ["Public"], summary: "Verify a window of the hash-chained decision ledger (resumable via afterSeq)", responses: { - 200: { description: "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." }, + 200: { description: "Window verified clean; nextAfterSeq is the resume cursor (null at the tip). Every response also carries tipSeq/tipHash/totalCount for third-party checkpointing, contentMismatches (#9850 -- the COUNT of rows whose preimage no longer matches the digest the chain committed to; a content mismatch no longer aborts the scan, because one unreconcilable row used to hide every row after it, so a structural break further along was unreachable. The verdict is unchanged: any mismatch is still ok:false and the first is still reported as `break`), and prunedRecords — the count of rows whose record preimage was legitimately pruned by the published retention window (chain checks still hold for them; only the content re-check is impossible, and the committed digest stays published)." }, 409: { description: "First break found: sequence_gap | predecessor_mismatch | row_hash_mismatch | missing_record | content_mismatch | short_tail (a record newer than the verified tip has no chain entry — the truncated-tail signature) | unchained_record (an INTERIOR record has no chain entry — the failed-append signature). Records younger than the 5-minute append grace window are not reported: the record insert and its chain append are two writes moments apart, and a verify landing between them is not evidence of tampering." }, }, }); diff --git a/src/review/decision-record.ts b/src/review/decision-record.ts index 3584396a92..bc8a1dfbbf 100644 --- a/src/review/decision-record.ts +++ b/src/review/decision-record.ts @@ -443,12 +443,28 @@ export async function verifyDecisionLedger( env: Env, afterSeq = 0, limit = 500, -): Promise<{ ok: boolean; checked: number; nextAfterSeq: number | null; tipSeq: number; tipHash: string; totalCount: number; prunedRecords: number; break?: LedgerBreak }> { +): Promise<{ ok: boolean; checked: number; nextAfterSeq: number | null; tipSeq: number; tipHash: string; totalCount: number; prunedRecords: number; contentMismatches: number; break?: LedgerBreak }> { const bounded = Math.max(1, Math.min(1000, limit)); // #9474: rows whose record preimage was legitimately pruned by the published retention policy (see the // missing-record branch below). Surfaced in the result so "the chain is clean but N old preimages are no // longer independently checkable" is an explicit, countable statement rather than silent. let prunedRecords = 0; + // #9850: content mismatches no longer ABORT the scan, they accumulate. Returning at the first one made a + // single unreconcilable row a denial-of-verification for everything after it: found on a live instance + // carrying 83 rows (seq 5-257) from before #9123 replaced the record-overwriting UPDATE with the revision + // scheme, where verification stopped at seq 5 and never examined the remaining 1,649 rows. Real tampering + // at seq 900 would have been invisible behind permanent, historical damage at seq 5. + // + // Safe to continue precisely BECAUSE the chain checks above already passed for this row: sequence, + // predecessor and row_hash all reconciled, so `prevHash` for the next row is sound. A content mismatch says + // "this row's preimage no longer matches what the chain committed to", which is a statement about that row + // alone. A STRUCTURAL break is different -- a sequence gap or predecessor mismatch means everything after + // it is unverifiable, so those still return immediately. + // + // `ok` is still false and the first mismatch is still reported as `break`, so nothing about the verdict is + // softened; the scan simply keeps going and reports how many there are. + let contentMismatches = 0; + let firstContentMismatch: LedgerBreak | null = null; const decisionRecordsPruneCutoff = retentionCutoffIsoForTable("decision_records"); const [totalRow, globalTip, prior] = await Promise.all([ env.DB.prepare("SELECT COUNT(*) AS n FROM decision_ledger").first<{ n: number }>(), @@ -461,7 +477,7 @@ export async function verifyDecisionLedger( const tipSeq = globalTip?.seq ?? 0; const tipHash = globalTip?.rowHash ?? LEDGER_GENESIS_HASH; // `== null` deliberately: D1 drivers disagree on .first() returning null vs undefined for no-row. - if (afterSeq > 0 && prior == null) return { ok: false, checked: 0, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "sequence_gap", atSeq: afterSeq, expectedSeq: afterSeq } }; + if (afterSeq > 0 && prior == null) return { ok: false, checked: 0, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "sequence_gap", atSeq: afterSeq, expectedSeq: afterSeq } }; let prevHash = prior?.rowHash ?? LEDGER_GENESIS_HASH; let expectedSeq = afterSeq + 1; const { results } = await env.DB.prepare( @@ -487,10 +503,10 @@ export async function verifyDecisionLedger( // from) so a call that finds ZERO new rows still has an anchor to reconcile against. let lastVerifiedCreatedAt = prior?.createdAt ?? null; for (const row of results) { - if (row.seq !== expectedSeq) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "sequence_gap", atSeq: row.seq, expectedSeq } }; - if (row.prevHash !== prevHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "predecessor_mismatch", atSeq: row.seq } }; + if (row.seq !== expectedSeq) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "sequence_gap", atSeq: row.seq, expectedSeq } }; + if (row.prevHash !== prevHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "predecessor_mismatch", atSeq: row.seq } }; const recomputed = await ledgerRowHash(prevHash, { seq: row.seq, recordId: row.recordId, recordDigest: row.recordDigest, createdAt: row.createdAt }); - if (recomputed !== row.rowHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "row_hash_mismatch", atSeq: row.seq } }; + if (recomputed !== row.rowHash) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "row_hash_mismatch", atSeq: row.seq } }; // #9078: the promised record/ledger reconciliation — a row_hash chained cleanly can still commit to a // digest whose CONTENT has since been rewritten (or whose preimage is simply gone). Neither is visible to // the chain-only checks above, since those only ever compare ledger rows against each other. @@ -508,7 +524,7 @@ export async function verifyDecisionLedger( if (decisionRecordsPruneCutoff !== null && row.createdAt < decisionRecordsPruneCutoff) { prunedRecords += 1; } else { - return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "missing_record", atSeq: row.seq, recordId: row.recordId } }; + return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: { kind: "missing_record", atSeq: row.seq, recordId: row.recordId } }; } } else { let recomputedContentDigest: string | null = null; @@ -518,7 +534,10 @@ export async function verifyDecisionLedger( // Unparseable record_json is itself proof the content no longer matches whatever the chain committed to. recomputedContentDigest = null; } - if (recomputedContentDigest !== row.recordDigest) return { ok: false, checked, nextAfterSeq: null, tipSeq, tipHash, totalCount, prunedRecords, break: { kind: "content_mismatch", atSeq: row.seq, recordId: row.recordId } }; + if (recomputedContentDigest !== row.recordDigest) { + contentMismatches += 1; + firstContentMismatch ??= { kind: "content_mismatch", atSeq: row.seq, recordId: row.recordId }; + } } prevHash = row.rowHash; lastVerifiedCreatedAt = row.createdAt; @@ -569,6 +588,7 @@ export async function verifyDecisionLedger( tipHash, totalCount, prunedRecords, + contentMismatches, break: orphan.createdAt > lastVerifiedCreatedAt ? { kind: "short_tail", atSeq: expectedSeq - 1 } @@ -576,7 +596,12 @@ export async function verifyDecisionLedger( }; } } - return { ok: true, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords }; + // A content mismatch is still a FAILED verification -- the scan continuing does not soften the verdict, it + // only stops one bad row from hiding every row after it. The first is reported as `break` exactly as before. + if (firstContentMismatch !== null) { + return { ok: false, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches, break: firstContentMismatch }; + } + return { ok: true, checked, nextAfterSeq, tipSeq, tipHash, totalCount, prunedRecords, contentMismatches }; } /** One ledger row, exactly as chained -- the shape `GET /v1/public/decision-ledger/row/:seq` returns. */ diff --git a/test/unit/decision-record.test.ts b/test/unit/decision-record.test.ts index 76cce78e10..f5bd9b29e8 100644 --- a/test/unit/decision-record.test.ts +++ b/test/unit/decision-record.test.ts @@ -384,7 +384,7 @@ describe("decision ledger (#8837)", () => { it("verifying a completely empty ledger returns ok:true with a zero tip, and skips the tail-truncation check (nothing to anchor against yet)", async () => { const env = createTestEnv(); const verified = await verifyDecisionLedger(env); - expect(verified).toEqual({ ok: true, checked: 0, nextAfterSeq: null, tipSeq: 0, tipHash: LEDGER_GENESIS_HASH, totalCount: 0, prunedRecords: 0 }); + expect(verified).toEqual({ ok: true, checked: 0, nextAfterSeq: null, tipSeq: 0, tipHash: LEDGER_GENESIS_HASH, totalCount: 0, prunedRecords: 0, contentMismatches: 0 }); }); it("TAIL TRUNCATION now breaks verify instead of passing clean (#9122): dropping the newest ledger rows leaves an orphaned decision_records tail", async () => { @@ -640,3 +640,76 @@ describe("verifier vs absence (#9474 pruned records, #9489 grace + interior orph expect(verified.break).toBeUndefined(); }); }); + +// #9850: a content mismatch used to ABORT verification, so one unreconcilable row was a denial-of- +// verification for every row after it. Found live: 83 rows (seq 5-257) left by the record-overwriting UPDATE +// that #9123 replaced, with verification stopping at seq 5 and never examining the remaining 1,649 rows -- +// real tampering at seq 900 would have been invisible behind permanent historical damage at seq 5. +describe("content mismatches do not mask later rows (#9850)", () => { + const seedChained = async (env: Env, count: number) => { + for (let i = 1; i <= count; i += 1) { + const { record, recordDigest } = await buildDecisionRecord(recordInput({ pullNumber: i })); + await persistDecisionRecord(env, record, recordDigest); + } + }; + /** Rewrite one record's stored body so its digest no longer matches what the chain committed to -- exactly + * the state the pre-#9123 UPDATE left behind. */ + const corruptRecordAt = async (env: Env, seq: number) => { + const row = await env.DB.prepare("SELECT record_id AS recordId FROM decision_ledger WHERE seq = ?").bind(seq).first<{ recordId: string }>(); + await env.DB.prepare("UPDATE decision_records SET record_json = ? WHERE id = ?").bind('{"tampered":true}', row!.recordId).run(); + }; + + it("REGRESSION: keeps verifying past a content mismatch instead of stopping at it", async () => { + const env = createTestEnv(); + await seedChained(env, 6); + await corruptRecordAt(env, 2); + + const verified = await verifyDecisionLedger(env); + + expect(verified.ok).toBe(false); + expect(verified.break).toMatchObject({ kind: "content_mismatch", atSeq: 2 }); + // The whole window was examined, not just the two rows before the break. + expect(verified.checked).toBe(6); + }); + + it("REGRESSION: a STRUCTURAL break after a content mismatch is still found -- it used to be unreachable", async () => { + // This is the security consequence, not just a cosmetic one: a historical unreconcilable row at seq 2 hid + // a genuinely broken chain further along. + const env = createTestEnv(); + await seedChained(env, 6); + await corruptRecordAt(env, 2); + await env.DB.prepare("UPDATE decision_ledger SET row_hash = ? WHERE seq = 5").bind("f".repeat(64)).run(); + + const verified = await verifyDecisionLedger(env); + + // The structural break wins the `break` slot: everything after it is unverifiable, so it is the more + // serious finding and the scan stops there. + expect(verified.break).toMatchObject({ kind: "row_hash_mismatch", atSeq: 5 }); + expect(verified.contentMismatches).toBe(1); // ...and the earlier content mismatch is still counted + }); + + it("counts EVERY content mismatch, not just the first", async () => { + const env = createTestEnv(); + await seedChained(env, 6); + for (const seq of [2, 4, 5]) await corruptRecordAt(env, seq); + + const verified = await verifyDecisionLedger(env); + + expect(verified.contentMismatches).toBe(3); + expect(verified.break).toMatchObject({ atSeq: 2 }); // the first is still what `break` names + }); + + it("INVARIANT: the verdict is not softened -- any mismatch still means ok:false", async () => { + const env = createTestEnv(); + await seedChained(env, 3); + await corruptRecordAt(env, 3); + expect((await verifyDecisionLedger(env)).ok).toBe(false); + }); + + it("reports contentMismatches: 0 on a clean chain, so the field is a fact and not only an error signal", async () => { + const env = createTestEnv(); + await seedChained(env, 4); + const verified = await verifyDecisionLedger(env); + expect(verified).toMatchObject({ ok: true, contentMismatches: 0 }); + }); +});