docs(storage): record the ingest authentication regime and its closeout evidence (#1384)

crates/graphforge-cli/tests/verify.rs

@@ -260,14 +260,20 @@ fn a_corrupted_reachable_object_is_refused_one_way_or_another() {
 
     let sha256_root = project.join("graph-objects").join("sha256");
     let mut corrupted = false;
-    for prefix in fs::read_dir(&sha256_root).unwrap() {
+    'outer: for prefix in fs::read_dir(&sha256_root).unwrap() {
         let prefix = prefix.unwrap().path();
         if !prefix.is_dir() {
             continue;
         }
         for object in fs::read_dir(&prefix).unwrap() {
             let object = object.unwrap().path();
             let mut bytes = fs::read(&object).unwrap();
+            // Directory order is not deterministic and a populated project can
+            // legitimately retain zero-byte objects; corrupt a non-empty one
+            // so the bit flip is real.
+            if bytes.is_empty() {
+                continue;
+            }
             bytes[0] ^= 0xFF;
             // Sealed CAS objects are mode 0444; reopen them writable first.
             let mut permissions = fs::metadata(&object).unwrap().permissions();
@@ -281,10 +287,7 @@ fn a_corrupted_reachable_object_is_refused_one_way_or_another() {
             fs::set_permissions(&object, permissions).unwrap();
             fs::write(&object, &bytes).unwrap();
             corrupted = true;
-            break;
-        }
-        if corrupted {
-            break;
+            break 'outer;
         }
     }
     assert!(

docs-site/astro.config.mjs

@@ -391,6 +391,10 @@ export default defineConfig({
                   label: '0044 — Research interchange preserves content identity separately from authority',
                   slug: 'adr/0044-research-interchange-authority',
                 },
+                {
+                  label: '0045 — Ingest authentication regime — hash once on write, verify at trust boundaries',
+                  slug: 'adr/0045-ingest-authentication-regime',
+                },
                 // END generated ADR records
               ],
             },

docs-site/scripts/sync-content.mjs

@@ -165,6 +165,7 @@ const PAGES = [
   'adr/0042-contextual-research-claims.md',
   'adr/0043-atomic-research-proposal-acceptance.md',
   'adr/0044-research-interchange-authority.md',
+  'adr/0045-ingest-authentication-regime.md',
   // END generated ADR records
   'releases/roadmap.md',
   'legal/licensing.md',

docs/adr/0045-ingest-authentication-regime.md

@@ -0,0 +1,155 @@
+---
+title: "ADR 0045: Ingest authentication regime — hash once on write, verify at trust boundaries"
+adr: "0045"
+status: "Accepted"
+date: "2026-09-22"
+superseded_by: null
+---
+
+# ADR 0045: Ingest authentication regime — hash once on write, verify at trust boundaries
+
+**Build target:** v0.6.0
+
+**Related:** ADR 0013 (project-generation protocol; the threat-model boundary),
+ADR 0018 (acknowledged durability and isolation), ADR 0038 (determinism at the
+publication boundary)
+
+## Context
+
+Measured at the 67,108,864-edge rung on `710c6c64`, ingest spent 310.1 GB of
+its 153.6 GB/310.1 GB read/write envelope almost entirely on authentication:
+99.1% of ingest reads existed to re-read and re-hash bytes this process had
+just written. Six of nine tracked storage I/O phases were authentication or
+verification. That is 17.3 times the final data size read purely to
+authenticate it, flat across a sixteen-fold scale range — a structural
+constant, not a scaling artifact. An external survey of eight comparable
+Arrow/Parquet/DataFusion systems (InfluxDB 3, GreptimeDB, Lance, Databend,
+SlateDB, ParadeDB, Iceberg/Delta, arrow-rs) found no system that reads data
+back in order to hash it; where verification exists it is a non-cryptographic
+checksum computed inline by the pass that produces the bytes.
+
+The maintainer decision of record (2026-09-17, issue #1384) reverses the
+burden of proof: every authentication pass must name a failure a user would
+notice that nothing else catches, or it is removed.
+
+One constraint shapes the result and is specific to how GraphForge deploys:
+the reference substrate is ext4 on an mdadm RAID 1 pair. ext4's
+`metadata_csum` covers inodes, directory blocks and the journal — it does not
+checksum file data, and RAID 1 mirrors blocks without checksums. Unlike the
+surveyed systems, GraphForge does not sit on an object store that verifies
+every put and get underneath it. So GraphForge verifies **once** rather than
+never; the cost being attacked is repetition and read-back, not the act of
+hashing each byte a single time.
+
+## Decision
+
+1. **Hash once, on write, streaming.** Every digest is produced by the same
+   pass that writes the bytes. No pass reads a payload back in order to hash
+   it.
+2. **Content-addressed naming stays cryptographic.** Where a digest names an
+   object — CAS graph objects, artifact receipts, authority digests — it is
+   SHA-256 and remains SHA-256. "Same name means same bytes" is a correctness
+   requirement for identity and deduplication; collision resistance is not
+   negotiable there. (BLAKE3 is additionally foreclosed for this role by FIPS
+   compliance obligations.)
+3. **Corruption detection uses a fast non-cryptographic checksum** for bytes
+   that are already named and whose producing pass is trusted: XXH64, seed 0,
+   implemented dependency-free in `crates/graphforge-storage/src/
+   corruption_checksum.rs` and recorded in `ArtifactReceipt::xxh64` beside
+   `sha256`. The threat is byte mutation, torn or partial writes, and silent
+   media corruption — the threat class of Parquet page checksums and ZFS
+   checksums — not a forged digest.
+4. **Prove-the-store-is-intact work runs nowhere in ingest.** It is reachable
+   through the explicit administrative `graphforge verify` command, which
+   validates the store on demand, as ParadeDB validates segment checksums on
+   demand.
+5. **The assumptions are recorded beside the code that depends on them** (in
+   `corruption_checksum.rs`, quoted below), not only in this ADR.
+
+### The two assumptions this rests on
+
+1. **The threat model excludes an active same-identity adversary**, per
+   ADR 0013's operational boundary. A non-cryptographic checksum is trivially
+   forgeable; it defends against accidental mutation, torn writes and media
+   corruption — not an attacker who can rewrite the receipt alongside the
+   payload. Untrusted shared storage, multi-tenant hosts or supply-chain
+   threats invalidate it.
+2. **The storage substrate does not checksum file data.** On a substrate that
+   does (ZFS, btrfs) the corruption check could relax further; on a weaker
+   one it must strengthen.
+
+If either stops holding, the decision reverts and the corruption checksum
+must be replaced by a cryptographic digest.
+
+### Determinism
+
+Digests stay reproducible for identical logical input: digest values are a
+pure function of the bytes each producing pass writes, and on-disk layout
+remains governed by ADR 0038. Removing read-back passes changes no digest
+value.
+
+## Surviving authentication boundaries
+
+Every authentication read that remains in the lifecycle, the failure it
+uniquely catches, and its measured cost at the 67,108,864-edge rung
+(4,194,304-edge rung in parentheses; application-I/O receipts, retained
+`b6ffb088` archive plus #1552):
+
+| Boundary | Mechanism | Failure it uniquely catches | Cost per edge |
+| --- | --- | --- | --- |
+| Producer write | SHA-256 computed inline by the pass writing the bytes (staged chunks, shaped outputs, canonical Parquet, CSR shards, CAS objects, receipts) | Digest is the product of the write; names the object | 0 extra reads (rides the write) |
+| Consume spool | SHA-256 fused into the authenticated source spool during encode | Shaped source mutated between shape completion and encode consumption | 0 extra reads (rides the consume) |
+| Shape replay/recovery | Identity + link count + length + XXH64 payload checksum (`authenticate_shaped_output`) | Same-inode, same-length mutation of a completed shape output — the #1269/#1392 class, regression-tested | One read of shaped outputs on replay paths only; 0 on a first ingest |
+| Resume recovery | `authenticate_artifact` over retained payloads at the checkpoint/resume boundary | Retained payload mutated while construction was interrupted — the #1269 class | 68.6 B/edge (62 B/edge) |
+| CAS install | Streaming copy re-derives SHA-256 and refuses a mismatch before the object becomes addressable | Torn or mutated artifact entering the content-addressed store; also names the object | Rides the copy (4.61 GB read at S22 is bytes moved into the store) |
+| CAS hydration (open time) | Streamed digest verification against the object's address | Store object no longer matches its name — silent media corruption | 5.1 B/edge (4.8 B/edge), charged at open, not ingest |
+| Shard read admission | Manifest digest + `codec::preflight` before Arrow decodes a CSR shard | Oversized or corrupt shard presented to the query path | Read-path work, outside ingest |
+
+`graphforge verify` re-derives every retained object's digest on demand and
+runs in no lifecycle path.
+
+## Removed passes
+
+The following read-back and re-hash passes were removed under #1384 and its
+focused repairs, each after recording why no surviving boundary needs it:
+the full SHA-256 re-hash of every consumed payload before supersession unlink;
+the postwrite re-read of encoded artifacts (#1444); the triple re-hash of the
+encoded inventory per ingest (#1450); the CSR shard write read-back (#1552);
+whole-graph re-verification on every property mutation (#1423);
+already-validated row re-hashing on provenance merge (#1419); per-object
+re-hashing in the storage-attribution walk (#1443); and the open-path
+generation inventory sweep re-hash (#1425).
+
+## Accounting note
+
+`ShapeConsumeReauthentication` and `EncodeWritePostwriteAuthentication` are
+**region names**, not authentication quantities: they are the thread I/O
+scopes of the shaping and encode regions respectively, and their reads are
+the partitioner's and encoder's real data movement. Authentication costs are
+attributed by the boundary table above, not by those phase names.
+
+## Consequences
+
+- Ingest application reads at the 67,108,864-edge rung fall from 310.1 GB
+  (authentication-dominated, 4,621 B/edge) to 68.5 GB
+  (data-movement-dominated), of which approximately 4.94 GB — 74 B/edge,
+  about 28% of the retained 265 B/edge — are authentication read-backs after
+  #1552 (5.85 GB including the 0.906 GB CSR shard read-back that #1552
+  removed). The shortfall against the issue's ~25 GB hashed-bytes target is
+  exactly the boundary table above, each entry named, justified and measured.
+- The #1269 corruption refusal holds unchanged, proven by its own regression
+  test, and is now established deliberately at the boundaries that consume
+  retained bytes.
+- Crash recovery, cancellation, corruption refusal and fail-closed publication
+  contracts are unchanged; all corruption, crash-boundary and recovery tests
+  pass unweakened.
+- ADR 0013 and ADR 0018 are unchanged; this record operates inside their
+  threat model and durability contract.
+
+## Evidence
+
+- Baseline measurement and survey: issue #1384 (2026-09-17).
+- Retained rung evidence: `docs/development/evidence/ladder/`
+  (the `b6ffb088` S18–S22 archive) and
+  `docs/development/evidence/authentication-regime-1384.md` for the
+  integrated-tree measurement that accompanied this record.

docs/adr/README.md

@@ -50,6 +50,7 @@ Roadmap-only ADRs are not retained in this tree.
 | 0042 | [Contextual research decisions extend immutable knowledge](0042-contextual-research-claims.md) | `0042-contextual-research-claims.md` |
 | 0043 | [Proposal acceptance shares the Project publication owner](0043-atomic-research-proposal-acceptance.md) | `0043-atomic-research-proposal-acceptance.md` |
 | 0044 | [Research interchange preserves content identity separately from authority](0044-research-interchange-authority.md) | `0044-research-interchange-authority.md` |
+| 0045 | [Ingest authentication regime — hash once on write, verify at trust boundaries](0045-ingest-authentication-regime.md) | `0045-ingest-authentication-regime.md` |
 
 ## Superseded records
 

docs/development/evidence/authentication-regime-1384.json

@@ -0,0 +1,45 @@
+{
+  "claim": "engineering_evidence_only",
+  "purpose": "#1384 closeout: the ingest authentication regime on the integrated tree",
+  "adr": "docs/adr/0045-ingest-authentication-regime.md",
+  "source_commit": "ed273d2b2558c4137a398e4980471ca3bfcac613",
+  "measured": "2026-09-22, OVHC-AGENCY, quiet host, stock release, native ladder S18-S19-S20 under delegated cgroup scope; archived with verified receipts under docs/development/evidence/ladder/ed273d2b2558c4137a398e4980471ca3bfcac613/",
+  "baseline_archive": "docs/development/evidence/ladder/b6ffb088405150afc117ba7180c876659cec729f/",
+  "throughput_rung_wall": {
+    "s18": {"edges": 4194304, "wall_seconds": 45.0, "edges_per_second": 93240},
+    "s19": {"edges": 8388608, "wall_seconds": 90.3, "edges_per_second": 92851},
+    "s20": {"edges": 16777216, "wall_seconds": 181.7, "edges_per_second": 92349},
+    "s22_baseline_b6ffb088": {"edges": 67108864, "wall_seconds": 725.7, "edges_per_second": 92470}
+  },
+  "ingest_application_io_gb": {
+    "s18": {"shape_region": [2.82, 1.78], "encode_region": [0.87, 0.26], "recovery_reauthentication": [0.26, 0.0], "cas_install": [0.26, 0.21], "hydration_verification": [0.02, 0.01], "append_merge": [0.0, 0.78]},
+    "s20": {"shape_region": [11.26, 7.11], "encode_region": [3.48, 1.11], "recovery_reauthentication": [1.11, 0.0], "cas_install": [1.11, 0.89], "hydration_verification": [0.08, 0.04], "append_merge": [0.0, 3.13]},
+    "s22_baseline": {"shape_region": [45.03, 28.42], "encode_region": [13.91, 4.6], "recovery_reauthentication": [4.6, 0.0], "cas_install": [4.61, 3.7], "hydration_verification": [0.34, 0.17], "append_merge": [0.0, 12.51]}
+  },
+  "authentication_read_back_bytes_at_s22": {
+    "before_1552_gb": 5.85,
+    "after_1552_gb": 4.94,
+    "csr_shard_read_back_removed_gb": 0.906,
+    "per_edge_bytes_after_1552": 74,
+    "per_edge_bytes_before_issue": 4579,
+    "note": "Recovery reauthentication 4.60 GB + hydration verification 0.34 GB remain; hydration is charged at object open, not ingest. The 310.1 GB baseline was authentication-dominated reads on the pre-redesign tree."
+  },
+  "reproducibility": "Per-phase ingest application-I/O counters are byte-identical to the b6ffb088 baseline at S18 and S20; rung correctness, digest reconciliation and result digests pass unchanged.",
+  "write_attribution": {
+    "application_to_device_ratio": {"s18": 4.25, "s19": 4.27, "s20": 4.29, "s22": 4.72},
+    "owners": "Device writes are a consistent 4.3-4.7x application writes, owned by the shaping region's spill/merge write-back pattern and fsync writeback; publication is separately attributed by publication-attribution-1481.md and is not a write sink; the transient peak is attributed by the lifecycle owner-union receipt (#1415, #1481)."
+  },
+  "acceptance_criteria_map": {
+    "AC1": "310.1 GB to 68.5 GB ingest reads at S22; remaining authentication read-backs 74 B/edge, each named and justified in ADR 0045's boundary table",
+    "AC2": "completed_shape_boundary_refuses_same_inode_payload_corruption and shaping_recovery_refuses_same_inode_payload_corruption pass in the 1,277-test storage suite",
+    "AC3": "ADR 0045 surviving-boundary table with mechanism, refusal and measured cost per edge",
+    "AC4": "Resume-boundary reauthentication is the #1269 class, measured 68.6 B/edge at S22; the earlier mis-attribution to #1269's fix is corrected in issue comments and the ADR accounting note",
+    "AC5": "Write volume attributed: consistent 4.3-4.7x application-to-device ratio with named owners; publication and transient peak attributed by #1481/#1415",
+    "AC6": "corruption_checksum.rs: SHA-256 names, XXH64 detects corruption",
+    "AC7": "Both assumptions recorded in corruption_checksum.rs and ADR 0045",
+    "AC8": "graphforge verify (crates/graphforge-cli/src/verify_cli.rs) reachable out of band; runs in no ingest path",
+    "AC9": "This document: edges/s and per-edge reads at three scales",
+    "AC10": "Corruption, cancellation, crash-boundary and recovery suites pass unweakened; digests and application-I/O counters reproduce byte-identically",
+    "AC11": "docs/adr/0045-ingest-authentication-regime.md"
+  }
+}