From 543d3a13e5e1f99008c8d68fd65b336657f977a1 Mon Sep 17 00:00:00 2001 From: James Ross Date: Sat, 27 Jun 2026 22:05:30 -0700 Subject: [PATCH] feat(warp-core): add WAL recovery plan object --- CHANGELOG.md | 7 + crates/warp-core/src/causal_wal.rs | 223 +++++++++++++++++++++ crates/warp-core/tests/causal_wal_tests.rs | 159 ++++++++++++++- docs/topics/WAL.md | 6 + docs/workflows.md | 2 +- xtask/src/main.rs | 21 +- 6 files changed, 413 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 137d360b..dcd334b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,10 @@ locator, writer epoch, and recovery certificate evidence; missing manifests or unavailable locators produce typed projection obstructions instead of empty success. +- `warp-core` now exposes `WalRecoveryPlan` records that bootstrap from a + projected WAL root or storage manifest, record checkpoint posture, committed + replay suffix, tail posture, recovered index roots, retained-material posture, + and projected evidence posture without requiring graph WAL nodes as input. - `warp-core` can now materialize WAL projection records into deterministic WARP graph facts with root, writer epoch, segment, commit-anchor, and recovery certificate nodes plus typed graph edges suitable for WSC serialization. The @@ -76,6 +80,9 @@ `retained_reading_missing_payload_is_not_empty_success` witness, locking the app-safe missing-retention posture for reading payloads, reading envelopes, and retained receipt support. +- `cargo xtask test-slice durability-release` now includes the exact + `recovery_plan_bootstraps_from_wal_root` witness, locking recovery plan + bootstrap posture without using Cargo's slow package-level name filter. - `warp-core` trusted runtime hosts now configure runtime WAL through `TrustedRuntimeWalConfig`, including in-memory and filesystem-backed adapters. `TrustedRuntimeWalStoreKind` exposes the configured adapter kind as diff --git a/crates/warp-core/src/causal_wal.rs b/crates/warp-core/src/causal_wal.rs index 41f2fafd..9fd3f1a3 100644 --- a/crates/warp-core/src/causal_wal.rs +++ b/crates/warp-core/src/causal_wal.rs @@ -3200,6 +3200,229 @@ impl WalRecoveryProjection { } } +/// Bootstrap evidence used to start a recovery plan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WalRecoveryBootstrapSource { + /// Recovery started from an already projected WAL root. + WalRoot { + /// Projection root digest. + root_digest: Hash, + /// Stable identity digest of the projected root. + root_identity_digest: Hash, + }, + /// Recovery started from storage manifest evidence. + StorageManifest { + /// Published manifest digest. + manifest_digest: Hash, + }, +} + +impl WalRecoveryBootstrapSource { + /// Builds bootstrap evidence from a projected WAL root. + #[must_use] + pub fn from_wal_root(root: &WalRoot) -> Self { + Self::WalRoot { + root_digest: root.root_digest, + root_identity_digest: root.identity_digest(), + } + } + + /// Builds bootstrap evidence from a storage manifest. + #[must_use] + pub fn from_manifest(manifest: &WalManifest) -> Self { + Self::StorageManifest { + manifest_digest: manifest.manifest_digest, + } + } +} + +/// Checkpoint selection posture for a recovery plan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WalRecoveryCheckpointPosture { + /// Recovery did not use a checkpoint base. + NotSelected, + /// Recovery selected a checkpoint and records its validation posture. + Selected { + /// Selected checkpoint digest. + checkpoint_digest: Hash, + /// Validation posture for the selected checkpoint. + validation: CheckpointValidationPosture, + }, +} + +/// Committed WAL suffix replayed by a recovery plan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WalRecoveryReplaySuffix { + /// First committed LSN replayed. + pub first_lsn: Option, + /// Last committed LSN replayed. + pub last_lsn: Option, + /// Number of committed transactions replayed. + pub committed_transactions: u64, + /// Final committed transaction digest. + pub final_commit_digest: Option, +} + +impl WalRecoveryReplaySuffix { + /// Builds replay suffix evidence from a recovery scan report. + #[must_use] + pub fn from_report(report: &RecoveryScanReport) -> Self { + Self { + first_lsn: report.first_committed_lsn(), + last_lsn: report.last_committed_lsn(), + committed_transactions: len_u64(report.transactions.len()), + final_commit_digest: report.last_commit_digest(), + } + } +} + +/// Index roots produced by recovery. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WalRecoveryIndexRoots { + /// Recovered frontier root. + pub recovered_frontier_root: Hash, + /// Recovered index root. + pub recovered_indexes_root: Hash, +} + +impl WalRecoveryIndexRoots { + /// Builds index-root evidence from a recovery certificate. + #[must_use] + pub fn from_certificate(certificate: &RecoveryCertificate) -> Self { + Self { + recovered_frontier_root: certificate.recovered_frontier_root, + recovered_indexes_root: certificate.recovered_indexes_root, + } + } +} + +/// Retained-material availability posture for recovery. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum WalRecoveryRetainedMaterialAvailability { + /// All retained material required by the recovered index is available. + Available, + /// At least one retained material reference is missing, corrupt, or obstructed. + Obstructed, +} + +/// Retained-material posture recorded by a recovery plan. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WalRecoveryRetainedMaterialPosture { + /// Overall retained-material availability. + pub availability: WalRecoveryRetainedMaterialAvailability, + /// Number of recovered retained material references. + pub material_count: u64, + /// Number of recovered reading references. + pub reading_count: u64, + /// Number of retained-material obstructions. + pub obstruction_count: u64, +} + +impl WalRecoveryRetainedMaterialPosture { + /// Builds retained-material posture from recovered retained evidence. + #[must_use] + pub fn from_retention_index( + index: &RecoveredRetentionIndex, + available_material: &BTreeSet, + ) -> Self { + let obstruction_count = + len_u64(retained_material_obstructions(index, available_material).len()); + Self { + availability: if obstruction_count == 0 { + WalRecoveryRetainedMaterialAvailability::Available + } else { + WalRecoveryRetainedMaterialAvailability::Obstructed + }, + material_count: len_u64(index.material_by_digest.len()), + reading_count: len_u64(index.reading_by_id.len()), + obstruction_count, + } + } +} + +/// Recovery plan/report shape tying bootstrap, replay, retention, and projection evidence. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WalRecoveryPlan { + /// Evidence used to bootstrap the recovery plan. + pub bootstrap_source: WalRecoveryBootstrapSource, + /// Checkpoint selection and validation posture. + pub checkpoint_posture: WalRecoveryCheckpointPosture, + /// Committed suffix replayed after the bootstrap source. + pub replay_suffix: WalRecoveryReplaySuffix, + /// Tail posture observed during recovery. + pub tail_posture: RecoveryTailPosture, + /// Recovered frontier and index roots. + pub index_roots: WalRecoveryIndexRoots, + /// Retained-material availability posture. + pub retained_material_posture: WalRecoveryRetainedMaterialPosture, + /// Projection posture for graph-ready WAL evidence. + pub projected_evidence_posture: WalRecoveryProjectionPosture, + /// Number of typed projection obstructions. + pub projected_evidence_obstruction_count: u64, +} + +impl WalRecoveryPlan { + /// Builds a recovery plan that bootstraps from a projected WAL root. + #[must_use] + pub fn from_wal_root( + root: &WalRoot, + report: &RecoveryScanReport, + checkpoint_posture: WalRecoveryCheckpointPosture, + certificate: &RecoveryCertificate, + retained_material_posture: WalRecoveryRetainedMaterialPosture, + projection: &WalRecoveryProjection, + ) -> Self { + Self::from_bootstrap_source( + WalRecoveryBootstrapSource::from_wal_root(root), + report, + checkpoint_posture, + certificate, + retained_material_posture, + projection, + ) + } + + /// Builds a recovery plan that bootstraps from storage manifest evidence. + #[must_use] + pub fn from_manifest( + manifest: &WalManifest, + report: &RecoveryScanReport, + checkpoint_posture: WalRecoveryCheckpointPosture, + certificate: &RecoveryCertificate, + retained_material_posture: WalRecoveryRetainedMaterialPosture, + projection: &WalRecoveryProjection, + ) -> Self { + Self::from_bootstrap_source( + WalRecoveryBootstrapSource::from_manifest(manifest), + report, + checkpoint_posture, + certificate, + retained_material_posture, + projection, + ) + } + + fn from_bootstrap_source( + bootstrap_source: WalRecoveryBootstrapSource, + report: &RecoveryScanReport, + checkpoint_posture: WalRecoveryCheckpointPosture, + certificate: &RecoveryCertificate, + retained_material_posture: WalRecoveryRetainedMaterialPosture, + projection: &WalRecoveryProjection, + ) -> Self { + Self { + bootstrap_source, + checkpoint_posture, + replay_suffix: WalRecoveryReplaySuffix::from_report(report), + tail_posture: report.tail_posture, + index_roots: WalRecoveryIndexRoots::from_certificate(certificate), + retained_material_posture, + projected_evidence_posture: projection.posture, + projected_evidence_obstruction_count: len_u64(projection.obstructions.len()), + } + } +} + /// Materialized WARP graph facts for a projected WAL root. /// /// The graph is a read-only fact projection. It carries typed nodes, edges, and diff --git a/crates/warp-core/tests/causal_wal_tests.rs b/crates/warp-core/tests/causal_wal_tests.rs index f7a8fb14..2e690d4b 100644 --- a/crates/warp-core/tests/causal_wal_tests.rs +++ b/crates/warp-core/tests/causal_wal_tests.rs @@ -40,9 +40,11 @@ use warp_core::causal_wal::{ TransactionLocalIndex, WalAppendAuthority, WalBuildError, WalCommitAnchor, WalCommittedTransaction, WalDoctorPosture, WalDurabilityMode, WalManifest, WalProjectionGraphObservationPosture, WalReceiptCorrelationRecord, WalRecordKind, - WalRecoveryError, WalRecoveryIndexError, WalRecoveryProjectionObstruction, - WalRecoveryProjectionPosture, WalRecoverySegmentEvidence, WalReleaseReadinessGates, WalRoot, - WalSchemaLintError, WalSegmentId, WalSegmentRef, WalSegmentSealPosture, + WalRecoveryBootstrapSource, WalRecoveryCheckpointPosture, WalRecoveryError, + WalRecoveryIndexError, WalRecoveryPlan, WalRecoveryProjectionObstruction, + WalRecoveryProjectionPosture, WalRecoveryRetainedMaterialAvailability, + WalRecoveryRetainedMaterialPosture, WalRecoverySegmentEvidence, WalReleaseReadinessGates, + WalRoot, WalSchemaLintError, WalSegmentId, WalSegmentRef, WalSegmentSealPosture, WalSegmentStorageLocator, WalStoreError, WalStorePort, WalTickDecision, WalTransactionBuilder, WalTransactionId, WalTransactionKind, WalWriterEpoch, WriterEpoch, WriterEpochId, WriterEpochRequest, WAL_PROJECTION_GRAPH_RECOVERY_CERTIFICATE_EDGE_TYPE, @@ -915,6 +917,157 @@ fn wal_projection_from_recovery() { )); } +#[test] +fn recovery_plan_bootstraps_from_wal_root() { + let label = "recovery-plan"; + let dir = temp_wal_dir(label); + let mut store = must_ok(FilesystemWalStore::open(&dir, WalSegmentId::from_raw(1))); + let writer_epoch = must_ok(store.acquire_writer_epoch(writer_epoch_request())); + let material = retained_material( + label, + RetainedMaterialKind::ReadingPayload, + EvidenceMaterialPosture::Present, + ); + let reading = reading_ref(label, EvidenceMaterialPosture::Present); + + must_ok(store.append_transaction(durable_submission_transaction(label, Lsn::from_raw(0)))); + must_ok(store.append_transaction(durable_tick_transaction( + label, + Lsn::from_raw(2), + WalTickDecision::Applied, + ))); + let builder = builder( + transaction_id("tx:recovery-plan:retained-reading"), + Lsn::from_raw(5), + WalAppendAuthority::TrustedScheduler, + WalTransactionKind::SchedulerTick, + ); + must_ok( + store.append_transaction(must_ok(build_retained_reading_transaction( + builder, + std::slice::from_ref(&material), + reading, + vec![frontier( + AffectedFrontierKind::ReadingIndex, + "recovery-plan:reading:before", + "recovery-plan:reading:after", + )], + ))), + ); + must_ok(store.seal_segment(epoch_id(), WalSegmentId::from_raw(1))); + let last_commit_digest = must_some( + store + .read_commits() + .last() + .map(|commit| commit.commit_digest), + ); + let manifest = WalManifest { + manifest_digest: digest("recovery-plan:manifest"), + last_committed_lsn: Some(Lsn::from_raw(6)), + last_commit_digest: Some(last_commit_digest), + sealed_segment_count: 1, + }; + must_ok(store.publish_manifest(epoch_id(), manifest.clone())); + + let report = must_ok(recover_filesystem_store(&dir, RecoveryAccessMode::ReadOnly)); + let checkpoint_digest = digest("recovery-plan:checkpoint"); + let certificate = build_recovery_certificate( + &report, + Some(checkpoint_digest), + 0, + digest("recovery-plan:frontier"), + digest("recovery-plan:indexes"), + ); + let writer_epoch = WalWriterEpoch::from_writer_epoch(&writer_epoch); + let projection = project_filesystem_wal_recovery( + &dir, + &report, + std::slice::from_ref(&writer_epoch), + Some(&certificate), + ); + assert_eq!(projection.posture, WalRecoveryProjectionPosture::Present); + let root = must_some(projection.root.clone()); + let retention = must_ok(recover_retention_index(&report)); + let available_material = BTreeSet::from([material.material_digest]); + let retained_material_posture = + WalRecoveryRetainedMaterialPosture::from_retention_index(&retention, &available_material); + + let plan = WalRecoveryPlan::from_wal_root( + &root, + &report, + WalRecoveryCheckpointPosture::Selected { + checkpoint_digest, + validation: CheckpointValidationPosture::PublishedAndUsable, + }, + &certificate, + retained_material_posture, + &projection, + ); + + assert_eq!( + plan.bootstrap_source, + WalRecoveryBootstrapSource::WalRoot { + root_digest: root.root_digest, + root_identity_digest: root.identity_digest() + } + ); + assert_eq!( + plan.checkpoint_posture, + WalRecoveryCheckpointPosture::Selected { + checkpoint_digest, + validation: CheckpointValidationPosture::PublishedAndUsable + } + ); + assert_eq!(plan.replay_suffix.first_lsn, Some(Lsn::from_raw(0))); + assert_eq!(plan.replay_suffix.last_lsn, Some(Lsn::from_raw(6))); + assert_eq!(plan.replay_suffix.committed_transactions, 3); + assert_eq!( + plan.replay_suffix.final_commit_digest, + Some(last_commit_digest) + ); + assert_eq!(plan.tail_posture, RecoveryTailPosture::Clean); + assert_eq!( + plan.index_roots.recovered_frontier_root, + certificate.recovered_frontier_root + ); + assert_eq!( + plan.index_roots.recovered_indexes_root, + certificate.recovered_indexes_root + ); + assert_eq!( + plan.retained_material_posture.availability, + WalRecoveryRetainedMaterialAvailability::Available + ); + assert_eq!(plan.retained_material_posture.material_count, 1); + assert_eq!(plan.retained_material_posture.reading_count, 1); + assert_eq!(plan.retained_material_posture.obstruction_count, 0); + assert_eq!( + plan.projected_evidence_posture, + WalRecoveryProjectionPosture::Present + ); + assert_eq!(plan.projected_evidence_obstruction_count, 0); + + let manifest_plan = WalRecoveryPlan::from_manifest( + &manifest, + &report, + WalRecoveryCheckpointPosture::NotSelected, + &certificate, + retained_material_posture, + &projection, + ); + assert_eq!( + manifest_plan.bootstrap_source, + WalRecoveryBootstrapSource::StorageManifest { + manifest_digest: manifest.manifest_digest + } + ); + assert_eq!( + manifest_plan.checkpoint_posture, + WalRecoveryCheckpointPosture::NotSelected + ); + assert_eq!(manifest_plan.replay_suffix, plan.replay_suffix); +} + #[test] fn wsc_ref_only_export_preserves_wal_identity() { let label = "wsc-ref-only"; diff --git a/docs/topics/WAL.md b/docs/topics/WAL.md index e09e7a4e..18ad9f16 100644 --- a/docs/topics/WAL.md +++ b/docs/topics/WAL.md @@ -118,6 +118,12 @@ A WAL path is a storage locator, not causal identity. Causal identity comes from the writer epoch, LSN range, segment digest, commit digest chain, and validated commit anchors. +Recovery planning records the bootstrap source, optional checkpoint posture, +committed replay suffix, tail posture, recovered index roots, retained-material +posture, and projected evidence posture. A recovery plan may start from a +projected WAL root or storage manifest, but it does not require graph WAL nodes +as recovery input. + ## Evidence The runtime ACK and recovery witnesses live in diff --git a/docs/workflows.md b/docs/workflows.md index 2b6cd8f4..67b89f08 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -164,7 +164,7 @@ The repo also exposes maintenance commands via `cargo xtask …`: - `cargo xtask test-slice contract-path-release` runs the v0.1 local contract-host release witness: installed contract pipeline replay, reference trusted host loop, and the serious external consumer fixture. - `cargo xtask test-slice runtime-wal-ack` runs the fast runtime WAL-backed ACK witness: app-facing acceptance rollback, scheduler tick receipt invariant checks, scheduler tick commit-before-publish, recovered indexes, CLI submission posture JSON, stale-claim guard, and generated man-page check. - `cargo xtask test-slice durable-runtime-wal` runs the release-grade filesystem runtime WAL durability witness: filesystem ACK recovery, filesystem failure atomicity, CLI submission posture JSON, stale-claim guard, and generated man-page check. -- `cargo xtask test-slice durability-release` runs the joined WAL/WSC release witness: filesystem runtime WAL durability, WSC retained evidence recovery, app-safe missing-retention posture, WSC topology recovery, topology WAL recovery, typed missing-material obstruction, stale-claim guards, doctrine checks, and generated man-page freshness. This is a release-gate slice, not the fastest local edit loop. +- `cargo xtask test-slice durability-release` runs the joined WAL/WSC release witness: filesystem runtime WAL durability, WSC retained evidence recovery, app-safe missing-retention posture, recovery plan bootstrap posture, WSC topology recovery, topology WAL recovery, typed missing-material obstruction, stale-claim guards, doctrine checks, and generated man-page freshness. This is a release-gate slice, not the fastest local edit loop. - `cargo xtask pr-preflight` runs the default changed-scope pre-PR gate against `origin/main`. - `cargo xtask pr-preflight --full` runs the broader explicit full pre-PR gate. - `cargo xtask dind` runs the DIND (Deterministic Ironclad Nightmare Drills) harness locally. diff --git a/xtask/src/main.rs b/xtask/src/main.rs index fabe7286..57c9b57a 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -801,6 +801,14 @@ fn build_test_slice_commands(slice: TestSlice) -> Vec { "retained_evidence_ref_tests", "retained_reading_missing_payload_is_not_empty_success", ]), + cargo_command([ + "test", + "-p", + "warp-core", + "--test", + "causal_wal_tests", + "recovery_plan_bootstraps_from_wal_root", + ]), cargo_command([ "test", "-p", @@ -6719,7 +6727,7 @@ mod tests { #[test] fn test_slice_durability_release_stays_explicit() { let commands = build_test_slice_commands(TestSlice::DurabilityRelease); - assert_eq!(commands.len(), 13); + assert_eq!(commands.len(), 14); let expected = [ ( @@ -6825,6 +6833,17 @@ mod tests { "retained_reading_missing_payload_is_not_empty_success", ], ), + ( + "cargo", + vec![ + "test", + "-p", + "warp-core", + "--test", + "causal_wal_tests", + "recovery_plan_bootstraps_from_wal_root", + ], + ), ( "cargo", vec![