diff --git a/CHANGELOG.md b/CHANGELOG.md index f913743c..fe2304f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,15 +46,20 @@ without making CAS semantic authority, reports missing blobs as typed import obstructions, and keeps equal bytes under different semantic coordinates as distinct retained material references. +- `warp-core` WAL WSC exports now carry retained material and reading-reference + envelopes through ref-only, self-contained, and CAS-addressed profiles. + Self-contained exports can embed retained payload bytes and validate them + against the WAL-retained material digest, while CAS-addressed imports require + the referenced retained blobs to be present before reporting success. - `warp-core` now includes a filesystem-backed WSC store adapter that persists envelope material separately from commit markers, hides staged material until marker publication, reopens committed envelopes in deterministic order, and reports torn envelope or marker files as typed WSC store obstructions. - `echo-cli` now exposes read-only `wsc causal-history` commands that export ref-only and self-contained WAL WSC bundles from filesystem WAL roots, inspect - bundle envelope metadata, verify self-contained bundles without the original - WAL root, and report unavailable ref-only segment bytes as typed material - obstructions in JSON output. + bundle envelope metadata including retained evidence envelopes, verify + self-contained bundles without the original WAL root, and report unavailable + ref-only segment bytes as typed material obstructions in JSON output. - `echo-cas` now exposes a fallible filesystem-backed `DiskTier` for durable retained blobs, preserving content-only BLAKE3 hash semantics across process reconstruction while keeping missing blobs as explicit absence. @@ -63,6 +68,10 @@ filesystem failure atomicity, CLI submission posture JSON, stale-claim, and generated man-page checks while leaving `runtime-wal-ack` as the fast semantic gate. +- `cargo xtask test-slice durability-release` now includes the exact + `wsc_retained_evidence_export_modes` witness, keeping retained-evidence WSC + export coverage in the release gate without using Cargo's slow package-level + name filter. - `warp-core` trusted runtime hosts now configure runtime WAL through `TrustedRuntimeWalConfig`, including in-memory and filesystem-backed adapters. `TrustedRuntimeWalStoreKind` exposes the configured adapter kind as @@ -695,6 +704,9 @@ Applied, Rejected, Obstructed}` with receipt evidence and typed contract - Echo 1.0 planning references now point at the cross-repository Continuum Stack Convergence Project instead of the retired Echo-only Project. +- Local `scripts/verify-local.sh full` now treats broad Cargo test lanes as + GitHub Actions-owned by default, while keeping a maintainer opt-in through + `VERIFY_LOCAL_FULL_TESTS=1` for intentional local full-suite runs. - `warp-core` renamed the generated contract package host API from `install_contract_package(...)` to `register_contract_package(...)` so the trusted-runtime boundary reads as explicit runtime-owned registration instead diff --git a/crates/warp-cli/README.md b/crates/warp-cli/README.md index aecdd8ff..599d4b43 100644 --- a/crates/warp-cli/README.md +++ b/crates/warp-cli/README.md @@ -77,12 +77,12 @@ history into Echo. Export commands inspect a filesystem WAL root read-only and require explicit writer-epoch projection evidence as JSON. ```sh -# Export graph facts plus external segment-byte dependencies +# Export graph facts plus retained evidence refs and external segment-byte dependencies echo-cli wsc causal-history export-ref-only runtime.wal \ --writer-epochs writer-epochs.json \ --out causal-history.ref-only -# Export graph facts plus embedded WAL segment bytes +# Export graph facts plus retained evidence refs and embedded WAL segment bytes echo-cli wsc causal-history export-self-contained runtime.wal \ --writer-epochs writer-epochs.json \ --out causal-history.self-contained @@ -98,6 +98,11 @@ Ref-only verification keeps the bundle observable but reports unavailable segment bytes as typed JSON material obstructions until external WAL material is provided by a future adapter. +Bundle manifests include `retained-evidence.ecwsc` for retained material and +reading references. Self-contained bundles also include `retained-payloads.ecwsc` +when retained payload bytes are embedded for import-time reveal and digest +validation. + ## Global Flags - `--format text|json` — Output format (default: `text`). Can appear before or after the subcommand. diff --git a/crates/warp-cli/src/wsc.rs b/crates/warp-cli/src/wsc.rs index a13a8a6b..7887e835 100644 --- a/crates/warp-cli/src/wsc.rs +++ b/crates/warp-cli/src/wsc.rs @@ -8,19 +8,19 @@ use std::path::{Path, PathBuf}; use anyhow::{bail, Context, Result}; use serde::{Deserialize, Serialize}; use warp_core::causal_wal::{ - canonical_segment_path, project_filesystem_wal_recovery, recover_filesystem_store, Lsn, - RecoveryAccessMode, RecoveryCertificateRef, RecoveryTailPosture, SubmissionAcceptanceRecord, - TickReceiptRecord, WalCommitAnchor, WalProjectionGraphObservationPosture, - WalReceiptCorrelationRecord, WalRecordKind, WalRecoveryProjectionPosture, WalRoot, - WalSegmentId, WalSegmentSealPosture, WalSegmentStorageLocator, WalTransactionId, - WalWriterEpoch, WriterEpochId, + canonical_segment_path, project_filesystem_wal_recovery, recover_filesystem_store, + recover_retention_index, Lsn, ReadingRefRecord, RecoveryAccessMode, RecoveryCertificateRef, + RecoveryTailPosture, RetainedMaterialRecord, SubmissionAcceptanceRecord, TickReceiptRecord, + WalCommitAnchor, WalProjectionGraphObservationPosture, WalReceiptCorrelationRecord, + WalRecordKind, WalRecoveryProjectionPosture, WalRoot, WalSegmentId, WalSegmentSealPosture, + WalSegmentStorageLocator, WalTransactionId, WalWriterEpoch, WriterEpochId, }; use warp_core::wsc::{ validate_wsc_cas_addressed_wal_export, validate_wsc_ref_only_wal_export, validate_wsc_self_contained_wal_export, wsc_ref_only_wal_export, wsc_self_contained_wal_export, WscCasAddressedWalExport, WscCasBlobStorePort, WscCausalHistoryExportProfileKind, WscRefOnlyWalExport, WscSelfContainedWalExport, WscSelfContainedWalSegmentMaterial, - WscStoreEnvelope, + WscStoreEnvelope, WscWalCausalHistoryRecords, }; use warp_core::Hash; @@ -31,6 +31,8 @@ const BUNDLE_MANIFEST: &str = "bundle.json"; const PROJECTION_ENVELOPE: &str = "projection.ecwsc"; const ACCEPTED_SUBMISSIONS_ENVELOPE: &str = "accepted-submissions.ecwsc"; const RECEIPT_CORRELATIONS_ENVELOPE: &str = "receipt-correlations.ecwsc"; +const RETAINED_EVIDENCE_ENVELOPE: &str = "retained-evidence.ecwsc"; +const RETAINED_PAYLOADS_ENVELOPE: &str = "retained-payloads.ecwsc"; const SEGMENT_MATERIAL_ENVELOPE: &str = "segment-material.ecwsc"; /// Exports a ref-only WAL causal-history WSC bundle. @@ -41,12 +43,7 @@ pub(crate) fn export_ref_only( format: &OutputFormat, ) -> Result<()> { let projected = project_wal_root(wal_root, writer_epochs)?; - let export = wsc_ref_only_wal_export( - &projected.root, - &projected.accepted_submissions, - &projected.receipts, - &projected.correlations, - )?; + let export = wsc_ref_only_wal_export(&projected.root, projected.records())?; let manifest = write_ref_only_bundle(out, &projected.root, &export)?; emit_export_report("export-ref-only", out, &manifest, format) @@ -64,9 +61,8 @@ pub(crate) fn export_self_contained( let export = wsc_self_contained_wal_export( &projected.root, &segment_materials, - &projected.accepted_submissions, - &projected.receipts, - &projected.correlations, + &[], + projected.records(), )?; let manifest = write_self_contained_bundle(out, &projected.root, &export)?; @@ -122,6 +118,20 @@ struct ProjectedWal { accepted_submissions: Vec, receipts: Vec, correlations: Vec, + retained_materials: Vec, + reading_refs: Vec, +} + +impl ProjectedWal { + fn records(&self) -> WscWalCausalHistoryRecords<'_> { + WscWalCausalHistoryRecords { + retained_materials: &self.retained_materials, + reading_refs: &self.reading_refs, + accepted_submissions: &self.accepted_submissions, + receipts: &self.receipts, + correlations: &self.correlations, + } + } } fn project_wal_root(wal_root: &Path, writer_epochs: &Path) -> Result { @@ -145,11 +155,14 @@ fn project_wal_root(wal_root: &Path, writer_epochs: &Path) -> Result Result { - let expected_dependencies = wsc_ref_only_wal_export(root, &[], &[], &[])?.segment_dependencies; + let expected_dependencies = + wsc_ref_only_wal_export(root, WscWalCausalHistoryRecords::empty())?.segment_dependencies; let export = WscRefOnlyWalExport { profile: WscCausalHistoryExportProfileKind::RefOnly, projection_envelope: read_envelope(bundle, &manifest.envelopes.projection)?, @@ -303,6 +328,7 @@ fn verify_ref_only( bundle, &manifest.envelopes.receipt_correlations, )?, + retention_envelope: read_envelope(bundle, &manifest.envelopes.retained_evidence)?, segment_dependencies: expected_dependencies, }; let imported = validate_wsc_ref_only_wal_export(&export, root)?; @@ -341,10 +367,16 @@ fn verify_self_contained( .segment_material .as_deref() .ok_or_else(|| anyhow::anyhow!("self-contained bundle is missing segment material"))?; + let retained_payloads = manifest + .envelopes + .retained_payloads + .as_deref() + .ok_or_else(|| anyhow::anyhow!("self-contained bundle is missing retained payloads"))?; let export = WscSelfContainedWalExport { profile: WscCausalHistoryExportProfileKind::SelfContained, projection_envelope: read_envelope(bundle, &manifest.envelopes.projection)?, segment_material_envelope: read_envelope(bundle, segment_material)?, + retained_material_envelope: read_envelope(bundle, retained_payloads)?, accepted_submission_envelope: read_envelope( bundle, &manifest.envelopes.accepted_submissions, @@ -353,6 +385,7 @@ fn verify_self_contained( bundle, &manifest.envelopes.receipt_correlations, )?, + retention_envelope: read_envelope(bundle, &manifest.envelopes.retained_evidence)?, }; let imported = validate_wsc_self_contained_wal_export(&export, root)?; Ok(verify_output( @@ -392,6 +425,7 @@ fn verify_cas_addressed( bundle, &manifest.envelopes.receipt_correlations, )?, + retention_envelope: read_envelope(bundle, &manifest.envelopes.retained_evidence)?, }; match validate_wsc_cas_addressed_wal_export(&export, root, &UnavailableCasStore) { Ok(imported) => Ok(verify_output( @@ -586,9 +620,12 @@ struct BundleEnvelopePaths { projection: String, accepted_submissions: String, receipt_correlations: String, + retained_evidence: String, #[serde(skip_serializing_if = "Option::is_none")] segment_material: Option, #[serde(skip_serializing_if = "Option::is_none")] + retained_payloads: Option, + #[serde(skip_serializing_if = "Option::is_none")] cas_references: Option, } @@ -598,10 +635,14 @@ impl BundleEnvelopePaths { ("projection", self.projection.as_str()), ("accepted_submissions", self.accepted_submissions.as_str()), ("receipt_correlations", self.receipt_correlations.as_str()), + ("retained_evidence", self.retained_evidence.as_str()), ]; if let Some(path) = self.segment_material.as_deref() { paths.push(("segment_material", path)); } + if let Some(path) = self.retained_payloads.as_deref() { + paths.push(("retained_payloads", path)); + } if let Some(path) = self.cas_references.as_deref() { paths.push(("cas_references", path)); } diff --git a/crates/warp-cli/tests/cli_integration.rs b/crates/warp-cli/tests/cli_integration.rs index d7eac7e1..67f767f9 100644 --- a/crates/warp-cli/tests/cli_integration.rs +++ b/crates/warp-cli/tests/cli_integration.rs @@ -759,7 +759,15 @@ fn wsc_causal_history_exports_and_verifies_profiles() -> TestResult { .success(); let inspect_json: serde_json::Value = serde_json::from_slice(&inspect.get_output().stdout)?; assert_eq!(inspect_json["profile"], "self-contained"); - assert_eq!(inspect_json["envelope_count"], 4); + assert_eq!(inspect_json["envelope_count"], 6); + let envelope_roles = inspect_json["envelopes"] + .as_array() + .ok_or("expected envelope summaries")? + .iter() + .map(|envelope| envelope["role"].as_str().ok_or("expected envelope role")) + .collect::, _>>()?; + assert!(envelope_roles.contains(&"retained_evidence")); + assert!(envelope_roles.contains(&"retained_payloads")); drop(fixture); diff --git a/crates/warp-core/src/wsc/mod.rs b/crates/warp-core/src/wsc/mod.rs index ab9d2cae..b1a08847 100644 --- a/crates/warp-core/src/wsc/mod.rs +++ b/crates/warp-core/src/wsc/mod.rs @@ -89,11 +89,11 @@ pub use store::{ WscReceiptCorrelationRecords, WscRefOnlyWalExport, WscRefOnlyWalExportError, WscRefOnlyWalImport, WscRefOnlyWalImportError, WscRefOnlyWalLocatorPosture, WscRefOnlyWalMaterialDependency, WscRefOnlyWalSegmentDependency, WscRetentionRecords, - WscSelfContainedWalExport, WscSelfContainedWalExportError, WscSelfContainedWalImport, - WscSelfContainedWalImportError, WscSelfContainedWalSegmentMaterial, WscStoreEnvelope, - WscStoreEnvelopeId, WscStoreObstruction, WscStoreObstructionKind, WscStorePort, - WscStoreRecordKind, WscStoreSubject, WscStoreWriteReceipt, WscTopologyRecords, - WSC_CAUSAL_HISTORY_EXPORT_PROFILE_VERSION, + WscSelfContainedRetainedMaterial, WscSelfContainedWalExport, WscSelfContainedWalExportError, + WscSelfContainedWalImport, WscSelfContainedWalImportError, WscSelfContainedWalSegmentMaterial, + WscStoreEnvelope, WscStoreEnvelopeId, WscStoreObstruction, WscStoreObstructionKind, + WscStorePort, WscStoreRecordKind, WscStoreSubject, WscStoreWriteReceipt, WscTopologyRecords, + WscWalCausalHistoryRecords, WSC_CAUSAL_HISTORY_EXPORT_PROFILE_VERSION, }; pub use validate::validate_wsc; pub use view::{AttachmentRef, WarpView, WscFile}; diff --git a/crates/warp-core/src/wsc/store.rs b/crates/warp-core/src/wsc/store.rs index cff159d8..e7d33624 100644 --- a/crates/warp-core/src/wsc/store.rs +++ b/crates/warp-core/src/wsc/store.rs @@ -82,6 +82,21 @@ const WSC_SELF_CONTAINED_WAL_SEGMENT_EDGE_TYPE: &str = "echo/wsc-store/wal-self-contained-segments/member/v1"; const WSC_SELF_CONTAINED_WAL_SEGMENT_ATTACHMENT_TYPE: &str = "echo/wsc-store/wal-self-contained-segments/segment-bytes/v1"; +const WSC_SELF_CONTAINED_RETAINED_BASIS_DOMAIN: &[u8] = + b"echo:wsc_store:self_contained_retained_basis:v1\0"; +const WSC_SELF_CONTAINED_RETAINED_NODE_DOMAIN: &[u8] = + b"echo:wsc_store:self_contained_retained_node:v1\0"; +const WSC_SELF_CONTAINED_RETAINED_EDGE_DOMAIN: &[u8] = + b"echo:wsc_store:self_contained_retained_edge:v1\0"; +const WSC_SELF_CONTAINED_RETAINED_SCHEMA: &str = "echo/wsc-store/self-contained-retained/v1"; +const WSC_SELF_CONTAINED_RETAINED_WARP: &str = "echo/wsc-store/self-contained-retained"; +const WSC_SELF_CONTAINED_RETAINED_ROOT: &str = "echo/wsc-store/self-contained-retained/root"; +const WSC_SELF_CONTAINED_RETAINED_NODE_TYPE: &str = + "echo/wsc-store/self-contained-retained/node/v1"; +const WSC_SELF_CONTAINED_RETAINED_EDGE_TYPE: &str = + "echo/wsc-store/self-contained-retained/member/v1"; +const WSC_SELF_CONTAINED_RETAINED_ATTACHMENT_TYPE: &str = + "echo/wsc-store/self-contained-retained/material-bytes/v1"; const WSC_CAS_ADDRESSED_WAL_REF_BASIS_DOMAIN: &[u8] = b"echo:wsc_store:cas_addressed_wal_ref_basis:v1\0"; const WSC_CAS_ADDRESSED_WAL_REF_NODE_DOMAIN: &[u8] = @@ -355,6 +370,35 @@ pub struct WscRefOnlyWalSegmentDependency { pub locator_posture: WscRefOnlyWalLocatorPosture, } +/// Record slices carried by WAL causal-history WSC exports. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct WscWalCausalHistoryRecords<'a> { + /// Retained material records recovered from WAL. + pub retained_materials: &'a [RetainedMaterialRecord], + /// Retained reading references recovered from WAL. + pub reading_refs: &'a [ReadingRefRecord], + /// Accepted submission records recovered from WAL. + pub accepted_submissions: &'a [SubmissionAcceptanceRecord], + /// Tick receipt records recovered from WAL. + pub receipts: &'a [TickReceiptRecord], + /// Receipt-correlation records recovered from WAL. + pub correlations: &'a [WalReceiptCorrelationRecord], +} + +impl WscWalCausalHistoryRecords<'_> { + /// Empty causal-history records for dependency-only export construction. + #[must_use] + pub const fn empty() -> Self { + Self { + retained_materials: &[], + reading_refs: &[], + accepted_submissions: &[], + receipts: &[], + correlations: &[], + } + } +} + /// Ref-only WAL causal-history WSC export. #[derive(Clone, Debug, PartialEq, Eq)] pub struct WscRefOnlyWalExport { @@ -366,6 +410,8 @@ pub struct WscRefOnlyWalExport { pub accepted_submission_envelope: WscStoreEnvelope, /// Tick receipt and receipt-correlation WSC envelope. pub receipt_correlation_envelope: WscStoreEnvelope, + /// Retained material and reading reference WSC envelope. + pub retention_envelope: WscStoreEnvelope, /// External segment byte dependencies for ref-only validation. pub segment_dependencies: Vec, } @@ -385,6 +431,8 @@ pub struct WscRefOnlyWalImport { pub receipts: Vec, /// Receipt-correlation records recovered from WSC. pub correlations: Vec, + /// Retained material and reading references recovered from WSC. + pub retention: WscRetentionRecords, /// External segment byte dependencies for ref-only validation. pub segment_dependencies: Vec, } @@ -398,6 +446,15 @@ pub struct WscSelfContainedWalSegmentMaterial { pub segment_bytes: Vec, } +/// Embedded retained payload material carried by a self-contained WSC export. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct WscSelfContainedRetainedMaterial { + /// Retained material record recovered from WAL. + pub material: RetainedMaterialRecord, + /// Raw retained payload bytes. + pub material_bytes: Vec, +} + /// Self-contained WAL causal-history WSC export. #[derive(Clone, Debug, PartialEq, Eq)] pub struct WscSelfContainedWalExport { @@ -407,10 +464,14 @@ pub struct WscSelfContainedWalExport { pub projection_envelope: WscStoreEnvelope, /// Embedded WAL segment material WSC envelope. pub segment_material_envelope: WscStoreEnvelope, + /// Embedded retained payload material WSC envelope. + pub retained_material_envelope: WscStoreEnvelope, /// Accepted submission evidence WSC envelope. pub accepted_submission_envelope: WscStoreEnvelope, /// Tick receipt and receipt-correlation WSC envelope. pub receipt_correlation_envelope: WscStoreEnvelope, + /// Retained material and reading reference WSC envelope. + pub retention_envelope: WscStoreEnvelope, } /// Imported and validated self-contained WAL causal-history WSC evidence. @@ -424,6 +485,8 @@ pub struct WscSelfContainedWalImport { pub root_identity_digest: Hash, /// Embedded segment recoveries validated against the projected WAL root. pub segment_recoveries: Vec, + /// Retained payload bytes recovered from embedded WSC material. + pub retained_payloads: Vec, /// Accepted submission records recovered from WSC. pub accepted_submissions: Vec, /// Tick receipt records recovered from WSC. @@ -434,6 +497,8 @@ pub struct WscSelfContainedWalImport { pub submission_index: RecoveredSubmissionIndex, /// Receipt correlation index rebuilt from imported WSC evidence. pub receipt_index: RecoveredReceiptIndex, + /// Retained material and reading references recovered from WSC. + pub retention: WscRetentionRecords, } /// CAS material for one WAL segment supplied by an exporter. @@ -507,6 +572,8 @@ pub struct WscCasAddressedWalExport { pub accepted_submission_envelope: WscStoreEnvelope, /// Tick receipt and receipt-correlation WSC envelope. pub receipt_correlation_envelope: WscStoreEnvelope, + /// Retained material and reading reference WSC envelope. + pub retention_envelope: WscStoreEnvelope, } /// Imported and validated CAS-addressed WAL causal-history WSC evidence. @@ -526,6 +593,8 @@ pub struct WscCasAddressedWalImport { pub receipts: Vec, /// Receipt-correlation records recovered from WSC. pub correlations: Vec, + /// Retained material and reading references recovered from WSC. + pub retention: WscRetentionRecords, } /// Minimal CAS blob lookup port used by CAS-addressed WSC validation. @@ -594,6 +663,9 @@ pub enum WscRefOnlyWalImportError { /// Receipt correlation WSC material was invalid. #[error("invalid receipt correlation WSC material")] ReceiptCorrelations(WscStoreObstruction), + /// Retained evidence WSC material was invalid. + #[error("invalid retained evidence WSC material")] + Retention(WscStoreObstruction), /// Causal-history WSC evidence was incomplete. #[error("incomplete causal-history WSC material")] IncompleteCausalHistory(WscStoreObstruction), @@ -617,15 +689,41 @@ pub enum WscSelfContainedWalExportError { /// Extra segment id. segment_id: WalSegmentId, }, + /// A present retained material record had no embedded payload bytes. + #[error("self-contained WAL WSC export is missing embedded retained material")] + MissingRetainedMaterial { + /// Missing retained material digest. + material_digest: Hash, + }, + /// Embedded retained material carried a digest absent from retained records. + #[error("self-contained WAL WSC export has extra embedded retained material")] + ExtraRetainedMaterial { + /// Extra retained material digest. + material_digest: Hash, + }, + /// Embedded retained material bytes did not hash to the retained digest. + #[error("self-contained WAL WSC retained material digest mismatch")] + RetainedMaterialDigestMismatch { + /// Expected retained material digest. + expected: Hash, + /// Actual retained material byte hash. + actual: Hash, + }, /// Embedded segment material WSC envelope was invalid. #[error("invalid self-contained WAL segment material WSC")] SegmentMaterial(WscStoreObstruction), + /// Embedded retained material WSC envelope was invalid. + #[error("invalid self-contained retained material WSC")] + RetainedMaterial(WscStoreObstruction), /// Accepted submission WSC envelope was invalid. #[error("invalid accepted submission WSC material")] AcceptedSubmissions(WscStoreObstruction), /// Receipt correlation WSC envelope was invalid. #[error("invalid receipt correlation WSC material")] ReceiptCorrelations(WscStoreObstruction), + /// Retained evidence WSC envelope was invalid. + #[error("invalid retained evidence WSC material")] + Retention(WscStoreObstruction), } /// Error returned when importing a self-contained WAL WSC export. @@ -737,12 +835,38 @@ pub enum WscSelfContainedWalImportError { /// Actual tail posture recovered from embedded bytes. actual: RecoveryTailPosture, }, + /// Embedded retained material WSC material was invalid. + #[error("invalid self-contained retained material WSC")] + RetainedMaterial(WscStoreObstruction), + /// A present retained material record had no embedded payload bytes. + #[error("self-contained WAL WSC import is missing embedded retained material")] + MissingRetainedMaterial { + /// Missing retained material digest. + material_digest: Hash, + }, + /// Embedded retained material carried a digest absent from retained records. + #[error("self-contained WAL WSC import has extra embedded retained material")] + ExtraRetainedMaterial { + /// Extra retained material digest. + material_digest: Hash, + }, + /// Embedded retained material bytes did not hash to the retained digest. + #[error("self-contained WAL WSC retained material digest mismatch")] + RetainedMaterialDigestMismatch { + /// Expected retained material digest. + expected: Hash, + /// Actual retained material byte hash. + actual: Hash, + }, /// Accepted submission WSC material was invalid. #[error("invalid accepted submission WSC material")] AcceptedSubmissions(WscStoreObstruction), /// Receipt correlation WSC material was invalid. #[error("invalid receipt correlation WSC material")] ReceiptCorrelations(WscStoreObstruction), + /// Retained evidence WSC material was invalid. + #[error("invalid retained evidence WSC material")] + Retention(WscStoreObstruction), /// Causal-history WSC evidence was incomplete. #[error("incomplete causal-history WSC material")] IncompleteCausalHistory(WscStoreObstruction), @@ -778,6 +902,9 @@ pub enum WscCasAddressedWalExportError { /// Receipt correlation WSC envelope was invalid. #[error("invalid receipt correlation WSC material")] ReceiptCorrelations(WscStoreObstruction), + /// Retained evidence WSC envelope was invalid. + #[error("invalid retained evidence WSC material")] + Retention(WscStoreObstruction), } /// Error returned when importing a CAS-addressed WAL WSC export. @@ -853,6 +980,9 @@ pub enum WscCasAddressedWalImportError { /// Receipt correlation WSC material was invalid. #[error("invalid receipt correlation WSC material")] ReceiptCorrelations(WscStoreObstruction), + /// Retained evidence WSC material was invalid. + #[error("invalid retained evidence WSC material")] + Retention(WscStoreObstruction), /// Causal-history WSC evidence was incomplete. #[error("incomplete causal-history WSC material")] IncompleteCausalHistory(WscStoreObstruction), @@ -1397,20 +1527,23 @@ impl WscTopologyRecords { /// one of the generated envelopes fails validation. pub fn wsc_ref_only_wal_export( root: &WalRoot, - accepted_submissions: &[SubmissionAcceptanceRecord], - receipts: &[TickReceiptRecord], - correlations: &[WalReceiptCorrelationRecord], + records: WscWalCausalHistoryRecords<'_>, ) -> Result { Ok(WscRefOnlyWalExport { profile: WscCausalHistoryExportProfileKind::RefOnly, projection_envelope: wsc_ref_only_wal_projection_envelope(root)?, accepted_submission_envelope: accepted_submission_records_to_wsc_envelope( - accepted_submissions, + records.accepted_submissions, ) .map_err(WscRefOnlyWalExportError::Envelope)?, receipt_correlation_envelope: receipt_correlation_records_to_wsc_envelope( - receipts, - correlations, + records.receipts, + records.correlations, + ) + .map_err(WscRefOnlyWalExportError::Envelope)?, + retention_envelope: retention_records_to_wsc_envelope( + records.retained_materials, + records.reading_refs, ) .map_err(WscRefOnlyWalExportError::Envelope)?, segment_dependencies: wsc_ref_only_wal_segment_dependencies(root)?, @@ -1477,6 +1610,8 @@ pub fn validate_wsc_ref_only_wal_export( &receipt_records.correlations, ) .map_err(WscRefOnlyWalImportError::IncompleteCausalHistory)?; + let retention = retention_records_from_wsc_envelope(&export.retention_envelope) + .map_err(WscRefOnlyWalImportError::Retention)?; Ok(WscRefOnlyWalImport { profile: export.profile, @@ -1485,6 +1620,7 @@ pub fn validate_wsc_ref_only_wal_export( accepted_submissions, receipts: receipt_records.receipts, correlations: receipt_records.correlations, + retention, segment_dependencies: expected_dependencies, }) } @@ -1502,13 +1638,18 @@ pub fn validate_wsc_ref_only_wal_export( pub fn wsc_self_contained_wal_export( root: &WalRoot, segment_materials: &[WscSelfContainedWalSegmentMaterial], - accepted_submissions: &[SubmissionAcceptanceRecord], - receipts: &[TickReceiptRecord], - correlations: &[WalReceiptCorrelationRecord], + retained_payloads: &[WscSelfContainedRetainedMaterial], + records: WscWalCausalHistoryRecords<'_>, ) -> Result { let segment_materials = canonical_self_contained_segment_materials(segment_materials) .map_err(WscSelfContainedWalExportError::SegmentMaterial)?; validate_self_contained_export_segment_material_ids(root, &segment_materials)?; + let retained_payloads = canonical_self_contained_retained_materials(retained_payloads) + .map_err(WscSelfContainedWalExportError::RetainedMaterial)?; + validate_self_contained_export_retained_payloads( + records.retained_materials, + &retained_payloads, + )?; Ok(WscSelfContainedWalExport { profile: WscCausalHistoryExportProfileKind::SelfContained, projection_envelope: wsc_ref_only_wal_projection_envelope(root) @@ -1517,15 +1658,24 @@ pub fn wsc_self_contained_wal_export( &segment_materials, ) .map_err(WscSelfContainedWalExportError::SegmentMaterial)?, + retained_material_envelope: self_contained_retained_materials_to_wsc_envelope( + &retained_payloads, + ) + .map_err(WscSelfContainedWalExportError::RetainedMaterial)?, accepted_submission_envelope: accepted_submission_records_to_wsc_envelope( - accepted_submissions, + records.accepted_submissions, ) .map_err(WscSelfContainedWalExportError::AcceptedSubmissions)?, receipt_correlation_envelope: receipt_correlation_records_to_wsc_envelope( - receipts, - correlations, + records.receipts, + records.correlations, ) .map_err(WscSelfContainedWalExportError::ReceiptCorrelations)?, + retention_envelope: retention_records_to_wsc_envelope( + records.retained_materials, + records.reading_refs, + ) + .map_err(WscSelfContainedWalExportError::Retention)?, }) } @@ -1577,6 +1727,12 @@ pub fn validate_wsc_self_contained_wal_export( .map_err(WscSelfContainedWalImportError::SegmentMaterial)?; let segment_recoveries = validate_self_contained_segment_recoveries(&segment_materials, expected_root)?; + let retained_payloads = + self_contained_retained_materials_from_wsc_envelope(&export.retained_material_envelope) + .map_err(WscSelfContainedWalImportError::RetainedMaterial)?; + let retention = retention_records_from_wsc_envelope(&export.retention_envelope) + .map_err(WscSelfContainedWalImportError::Retention)?; + validate_self_contained_import_retained_payloads(&retention.materials, &retained_payloads)?; let accepted_submissions = accepted_submission_records_from_wsc_envelope(&export.accepted_submission_envelope) @@ -1605,11 +1761,13 @@ pub fn validate_wsc_self_contained_wal_export( projection, root_identity_digest: expected_root_identity, segment_recoveries, + retained_payloads, accepted_submissions, receipts: receipt_records.receipts, correlations: receipt_records.correlations, submission_index, receipt_index, + retention, }) } @@ -1627,12 +1785,11 @@ pub fn validate_wsc_self_contained_wal_export( pub fn wsc_cas_addressed_wal_export( root: &WalRoot, segment_materials: &[WscCasAddressedWalSegmentMaterial], - retained_materials: &[WscCasAddressedRetainedMaterialReference], - accepted_submissions: &[SubmissionAcceptanceRecord], - receipts: &[TickReceiptRecord], - correlations: &[WalReceiptCorrelationRecord], + retained_material_references: &[WscCasAddressedRetainedMaterialReference], + records: WscWalCausalHistoryRecords<'_>, ) -> Result { - let references = wsc_cas_addressed_wal_references(root, segment_materials, retained_materials)?; + let references = + wsc_cas_addressed_wal_references(root, segment_materials, retained_material_references)?; Ok(WscCasAddressedWalExport { profile: WscCausalHistoryExportProfileKind::CasAddressed, projection_envelope: wsc_ref_only_wal_projection_envelope(root) @@ -1640,14 +1797,19 @@ pub fn wsc_cas_addressed_wal_export( cas_reference_envelope: cas_addressed_wal_references_to_wsc_envelope(&references) .map_err(WscCasAddressedWalExportError::CasReferences)?, accepted_submission_envelope: accepted_submission_records_to_wsc_envelope( - accepted_submissions, + records.accepted_submissions, ) .map_err(WscCasAddressedWalExportError::AcceptedSubmissions)?, receipt_correlation_envelope: receipt_correlation_records_to_wsc_envelope( - receipts, - correlations, + records.receipts, + records.correlations, ) .map_err(WscCasAddressedWalExportError::ReceiptCorrelations)?, + retention_envelope: retention_records_to_wsc_envelope( + records.retained_materials, + records.reading_refs, + ) + .map_err(WscCasAddressedWalExportError::Retention)?, }) } @@ -1716,6 +1878,8 @@ where &receipt_records.correlations, ) .map_err(WscCasAddressedWalImportError::IncompleteCausalHistory)?; + let retention = retention_records_from_wsc_envelope(&export.retention_envelope) + .map_err(WscCasAddressedWalImportError::Retention)?; Ok(WscCasAddressedWalImport { profile: export.profile, @@ -1725,6 +1889,7 @@ where accepted_submissions, receipts: receipt_records.receipts, correlations: receipt_records.correlations, + retention, }) } @@ -3113,6 +3278,170 @@ fn self_contained_segment_materials_from_wsc_envelope( Ok(materials) } +fn self_contained_retained_materials_to_wsc_envelope( + materials: &[WscSelfContainedRetainedMaterial], +) -> Result { + let materials = canonical_self_contained_retained_materials(materials)?; + let mut store = GraphStore::new(make_warp_id(WSC_SELF_CONTAINED_RETAINED_WARP)); + let root = make_node_id(WSC_SELF_CONTAINED_RETAINED_ROOT); + store.insert_node( + root, + NodeRecord { + ty: make_type_id(WSC_SELF_CONTAINED_RETAINED_NODE_TYPE), + }, + ); + for material in &materials { + let payload = self_contained_retained_material_payload(material); + let node = self_contained_retained_material_node_id(&payload); + store.insert_node( + node, + NodeRecord { + ty: make_type_id(WSC_SELF_CONTAINED_RETAINED_NODE_TYPE), + }, + ); + store.insert_edge( + root, + EdgeRecord { + id: self_contained_retained_material_edge_id(material.material.material_digest), + from: root, + to: node, + ty: make_type_id(WSC_SELF_CONTAINED_RETAINED_EDGE_TYPE), + }, + ); + store.set_node_attachment( + node, + Some(AttachmentValue::Atom(AtomPayload::new( + make_type_id(WSC_SELF_CONTAINED_RETAINED_ATTACHMENT_TYPE), + Bytes::from(payload), + ))), + ); + } + let basis_digest = self_contained_retained_material_basis_digest(&materials); + let input = build_one_warp_input(&store, root); + let wsc_bytes = write_wsc_one_warp( + &input, + make_type_id(WSC_SELF_CONTAINED_RETAINED_SCHEMA).0, + 0, + ) + .map_err(|_| WscStoreObstruction::invalid_wsc(basis_digest))?; + WscStoreEnvelope::validated( + WscStoreRecordKind::RetainedEvidence, + basis_digest, + wsc_bytes, + ) +} + +fn self_contained_retained_materials_from_wsc_envelope( + envelope: &WscStoreEnvelope, +) -> Result, WscStoreObstruction> { + if envelope.record_kind() != WscStoreRecordKind::RetainedEvidence { + return Err(WscStoreObstruction::invalid_envelope(0)); + } + let wsc_digest = *envelope.wsc_digest(); + let file = WscFile::from_bytes(envelope.wsc_bytes().to_vec()) + .map_err(|_| WscStoreObstruction::invalid_wsc(wsc_digest))?; + validate_wsc(&file).map_err(|_| WscStoreObstruction::invalid_wsc(wsc_digest))?; + if file.schema_hash() != &make_type_id(WSC_SELF_CONTAINED_RETAINED_SCHEMA).0 { + return Err(WscStoreObstruction::invalid_wsc(wsc_digest)); + } + let view = file + .warp_view(0) + .map_err(|_| WscStoreObstruction::invalid_wsc(wsc_digest))?; + let mut materials = Vec::new(); + for node_index in 0..view.nodes().len() { + for attachment in view.node_attachments(node_index) { + if attachment.type_or_warp + != make_type_id(WSC_SELF_CONTAINED_RETAINED_ATTACHMENT_TYPE).0 + { + return Err(WscStoreObstruction::invalid_wsc(wsc_digest)); + } + let payload = atom_payload_bytes(&view, attachment, wsc_digest)?; + materials.push(self_contained_retained_material_from_payload( + payload, wsc_digest, + )?); + } + } + let materials = canonical_self_contained_retained_materials(&materials)?; + let basis_digest = self_contained_retained_material_basis_digest(&materials); + if envelope.basis_digest() != &basis_digest { + return Err(WscStoreObstruction::basis_digest_mismatch( + *envelope.basis_digest(), + basis_digest, + )); + } + Ok(materials) +} + +fn validate_self_contained_export_retained_payloads( + retained_materials: &[RetainedMaterialRecord], + retained_payloads: &[WscSelfContainedRetainedMaterial], +) -> Result<(), WscSelfContainedWalExportError> { + let material_by_digest = retained_materials + .iter() + .map(|material| (material.material_digest, *material)) + .collect::>(); + validate_self_contained_retained_hashes(retained_payloads).map_err(|(expected, actual)| { + WscSelfContainedWalExportError::RetainedMaterialDigestMismatch { expected, actual } + })?; + for material in retained_materials + .iter() + .filter(|material| material.posture == crate::causal_wal::EvidenceMaterialPosture::Present) + { + if !retained_payloads + .iter() + .any(|payload| payload.material.material_digest == material.material_digest) + { + return Err(WscSelfContainedWalExportError::MissingRetainedMaterial { + material_digest: material.material_digest, + }); + } + } + if let Some(extra) = retained_payloads + .iter() + .find(|payload| !material_by_digest.contains_key(&payload.material.material_digest)) + { + return Err(WscSelfContainedWalExportError::ExtraRetainedMaterial { + material_digest: extra.material.material_digest, + }); + } + Ok(()) +} + +fn validate_self_contained_import_retained_payloads( + retained_materials: &[RetainedMaterialRecord], + retained_payloads: &[WscSelfContainedRetainedMaterial], +) -> Result<(), WscSelfContainedWalImportError> { + let material_by_digest = retained_materials + .iter() + .map(|material| (material.material_digest, *material)) + .collect::>(); + validate_self_contained_retained_hashes(retained_payloads).map_err(|(expected, actual)| { + WscSelfContainedWalImportError::RetainedMaterialDigestMismatch { expected, actual } + })?; + for material in retained_materials + .iter() + .filter(|material| material.posture == crate::causal_wal::EvidenceMaterialPosture::Present) + { + if !retained_payloads + .iter() + .any(|payload| payload.material.material_digest == material.material_digest) + { + return Err(WscSelfContainedWalImportError::MissingRetainedMaterial { + material_digest: material.material_digest, + }); + } + } + if let Some(extra) = retained_payloads + .iter() + .find(|payload| !material_by_digest.contains_key(&payload.material.material_digest)) + { + return Err(WscSelfContainedWalImportError::ExtraRetainedMaterial { + material_digest: extra.material.material_digest, + }); + } + Ok(()) +} + fn validate_self_contained_segment_recoveries( materials: &[WscSelfContainedWalSegmentMaterial], expected_root: &WalRoot, @@ -3232,6 +3561,25 @@ fn canonical_self_contained_segment_materials( Ok(by_segment.into_values().collect()) } +fn canonical_self_contained_retained_materials( + materials: &[WscSelfContainedRetainedMaterial], +) -> Result, WscStoreObstruction> { + let mut by_digest = BTreeMap::new(); + for material in materials { + if let Some(existing) = by_digest.get(&material.material.material_digest) { + if existing != material { + return Err(WscStoreObstruction::duplicate_mismatch( + self_contained_retained_material_duplicate_id( + material.material.material_digest, + ), + )); + } + } + by_digest.insert(material.material.material_digest, material.clone()); + } + Ok(by_digest.into_values().collect()) +} + fn self_contained_segment_material_payload( material: &WscSelfContainedWalSegmentMaterial, ) -> Vec { @@ -3276,6 +3624,36 @@ fn self_contained_segment_material_from_payload( }) } +fn self_contained_retained_material_payload( + material: &WscSelfContainedRetainedMaterial, +) -> Vec { + let material_record = material.material.to_payload_bytes(); + let mut out = Vec::new(); + out.extend_from_slice(&len_u64(material_record.len()).to_le_bytes()); + out.extend_from_slice(&material_record); + out.extend_from_slice(&len_u64(material.material_bytes.len()).to_le_bytes()); + out.extend_from_slice(&material.material_bytes); + out +} + +fn self_contained_retained_material_from_payload( + bytes: &[u8], + wsc_digest: Hash, +) -> Result { + let mut cursor = WscPayloadCursor::new(bytes, wsc_digest); + let record_len = cursor.read_usize()?; + let record_bytes = cursor.read_bytes(record_len)?; + let material = RetainedMaterialRecord::from_payload_bytes(record_bytes) + .map_err(|_| WscStoreObstruction::invalid_wsc(wsc_digest))?; + let material_len = cursor.read_usize()?; + let material_bytes = cursor.read_bytes(material_len)?.to_vec(); + cursor.finish()?; + Ok(WscSelfContainedRetainedMaterial { + material, + material_bytes, + }) +} + fn self_contained_segment_material_basis_digest( materials: &[WscSelfContainedWalSegmentMaterial], ) -> Hash { @@ -3287,6 +3665,17 @@ fn self_contained_segment_material_basis_digest( hasher.finalize().into() } +fn self_contained_retained_material_basis_digest( + materials: &[WscSelfContainedRetainedMaterial], +) -> Hash { + let mut hasher = Hasher::new(); + hasher.update(WSC_SELF_CONTAINED_RETAINED_BASIS_DOMAIN); + for material in materials { + hasher.update(&self_contained_retained_material_payload(material)); + } + hasher.finalize().into() +} + fn self_contained_segment_material_node_id(payload: &[u8]) -> NodeId { let mut hasher = Hasher::new(); hasher.update(WSC_SELF_CONTAINED_WAL_SEGMENT_NODE_DOMAIN); @@ -3294,6 +3683,13 @@ fn self_contained_segment_material_node_id(payload: &[u8]) -> NodeId { NodeId(hasher.finalize().into()) } +fn self_contained_retained_material_node_id(payload: &[u8]) -> NodeId { + let mut hasher = Hasher::new(); + hasher.update(WSC_SELF_CONTAINED_RETAINED_NODE_DOMAIN); + hasher.update(payload); + NodeId(hasher.finalize().into()) +} + fn self_contained_segment_material_edge_id(segment_id: WalSegmentId) -> EdgeId { let mut hasher = Hasher::new(); hasher.update(WSC_SELF_CONTAINED_WAL_SEGMENT_EDGE_DOMAIN); @@ -3301,6 +3697,13 @@ fn self_contained_segment_material_edge_id(segment_id: WalSegmentId) -> EdgeId { EdgeId(hasher.finalize().into()) } +fn self_contained_retained_material_edge_id(material_digest: Hash) -> EdgeId { + let mut hasher = Hasher::new(); + hasher.update(WSC_SELF_CONTAINED_RETAINED_EDGE_DOMAIN); + hasher.update(&material_digest); + EdgeId(hasher.finalize().into()) +} + fn self_contained_segment_material_duplicate_id(segment_id: WalSegmentId) -> WscStoreEnvelopeId { let mut hasher = Hasher::new(); hasher.update(WSC_SELF_CONTAINED_WAL_SEGMENT_NODE_DOMAIN); @@ -3309,6 +3712,26 @@ fn self_contained_segment_material_duplicate_id(segment_id: WalSegmentId) -> Wsc WscStoreEnvelopeId::from_hash(hasher.finalize().into()) } +fn self_contained_retained_material_duplicate_id(material_digest: Hash) -> WscStoreEnvelopeId { + let mut hasher = Hasher::new(); + hasher.update(WSC_SELF_CONTAINED_RETAINED_NODE_DOMAIN); + hasher.update(b"duplicate"); + hasher.update(&material_digest); + WscStoreEnvelopeId::from_hash(hasher.finalize().into()) +} + +fn validate_self_contained_retained_hashes( + materials: &[WscSelfContainedRetainedMaterial], +) -> Result<(), (Hash, Hash)> { + for material in materials { + let actual = cas_content_hash(&material.material_bytes); + if actual != material.material.material_digest { + return Err((material.material.material_digest, actual)); + } + } + Ok(()) +} + fn len_u64(len: usize) -> u64 { match u64::try_from(len) { Ok(value) => value, @@ -3834,6 +4257,19 @@ impl<'a> WscPayloadCursor<'a> { self.read_array::<32>() } + fn read_bytes(&mut self, len: usize) -> Result<&'a [u8], WscStoreObstruction> { + let end = self + .offset + .checked_add(len) + .ok_or_else(|| WscStoreObstruction::invalid_wsc(self.wsc_digest))?; + let slice = self + .bytes + .get(self.offset..end) + .ok_or_else(|| WscStoreObstruction::invalid_wsc(self.wsc_digest))?; + self.offset = end; + Ok(slice) + } + fn read_array(&mut self) -> Result<[u8; N], WscStoreObstruction> { let end = self .offset diff --git a/crates/warp-core/tests/causal_wal_tests.rs b/crates/warp-core/tests/causal_wal_tests.rs index a45d5696..f7a8fb14 100644 --- a/crates/warp-core/tests/causal_wal_tests.rs +++ b/crates/warp-core/tests/causal_wal_tests.rs @@ -59,9 +59,10 @@ use warp_core::wsc::{ InMemoryWscStore, WscCasAddressedRetainedMaterialReference, WscCasAddressedWalImportError, WscCasAddressedWalSegmentMaterial, WscCasBlobStorePort, WscCausalHistoryExportProfileKind, WscFile, WscRefOnlyWalExportError, WscRefOnlyWalLocatorPosture, - WscRefOnlyWalMaterialDependency, WscSelfContainedWalExportError, - WscSelfContainedWalImportError, WscSelfContainedWalSegmentMaterial, WscStoreEnvelope, - WscStoreObstructionKind, WscStorePort, WscStoreRecordKind, + WscRefOnlyWalMaterialDependency, WscSelfContainedRetainedMaterial, + WscSelfContainedWalExportError, WscSelfContainedWalImportError, + WscSelfContainedWalSegmentMaterial, WscStoreEnvelope, WscStoreObstructionKind, WscStorePort, + WscStoreRecordKind, WscWalCausalHistoryRecords, }; use warp_core::{ make_strand_id, make_type_id, AuthorityDomainId, AuthorityDomainRef, BraidEvent, BraidStatus, @@ -967,9 +968,7 @@ fn wsc_ref_only_export_preserves_wal_identity() { let export = must_ok(wsc_ref_only_wal_export( &root, - &[acceptance], - &[receipt], - &[correlation], + wsc_records(&[], &[], &[acceptance], &[receipt], &[correlation]), )); let imported = must_ok(validate_wsc_ref_only_wal_export(&export, &root)); @@ -1022,15 +1021,11 @@ fn wsc_ref_only_export_preserves_wal_identity() { let absolute_export_a = must_ok(wsc_ref_only_wal_export( &absolute_a, - &[acceptance], - &[receipt], - &[correlation], + wsc_records(&[], &[], &[acceptance], &[receipt], &[correlation]), )); let absolute_export_b = must_ok(wsc_ref_only_wal_export( &absolute_b, - &[acceptance], - &[receipt], - &[correlation], + wsc_records(&[], &[], &[acceptance], &[receipt], &[correlation]), )); assert_eq!( absolute_export_a.projection_envelope.basis_digest(), @@ -1066,7 +1061,10 @@ fn wsc_ref_only_export_preserves_wal_identity() { missing_locator.segments[0].storage_locator = None; assert_eq!( must_err( - wsc_ref_only_wal_export(&missing_locator, &[acceptance], &[receipt], &[correlation]), + wsc_ref_only_wal_export( + &missing_locator, + wsc_records(&[], &[], &[acceptance], &[receipt], &[correlation]), + ), "ref-only WSC exports require segment locators", ), WscRefOnlyWalExportError::MissingSegmentLocator { @@ -1130,7 +1128,12 @@ fn wsc_self_contained_export_replays_segment_bytes() { assert_eq!( must_err( - wsc_self_contained_wal_export(&root, &[], &[acceptance], &[receipt], &[correlation]), + wsc_self_contained_wal_export( + &root, + &[], + &[], + wsc_records(&[], &[], &[acceptance], &[receipt], &[correlation]), + ), "self-contained WSC exports require embedded segment material", ), WscSelfContainedWalExportError::MissingSegmentMaterial { @@ -1144,9 +1147,8 @@ fn wsc_self_contained_export_replays_segment_bytes() { segment_id: segment.segment_id, segment_bytes: segment_bytes.clone(), }], - &[acceptance], - &[receipt], - &[correlation], + &[], + wsc_records(&[], &[], &[acceptance], &[receipt], &[correlation]), )); drop(store); must_ok(fs::remove_dir_all(&dir)); @@ -1227,9 +1229,8 @@ fn wsc_self_contained_export_replays_segment_bytes() { segment_id: segment.segment_id, segment_bytes: tampered_segment_bytes, }], - &[acceptance], - &[receipt], - &[correlation], + &[], + wsc_records(&[], &[], &[acceptance], &[receipt], &[correlation]), )); let error = must_err( validate_wsc_self_contained_wal_export(&tampered_export, &root), @@ -1335,9 +1336,7 @@ fn wsc_cas_addressed_export_requires_present_blobs() { byte_len: retained_b.byte_len, }, ], - &[acceptance], - &[receipt], - &[correlation], + wsc_records(&[], &[], &[acceptance], &[receipt], &[correlation]), )); let imported = must_ok(validate_wsc_cas_addressed_wal_export( @@ -1396,6 +1395,186 @@ fn wsc_cas_addressed_export_requires_present_blobs() { )); } +#[test] +fn wsc_retained_evidence_export_modes() { + let label = "wsc-retained-evidence"; + let dir = temp_wal_dir(label); + let mut store = must_ok(FilesystemWalStore::open(&dir, WalSegmentId::from_raw(1))); + let writer_epoch = must_ok(store.acquire_writer_epoch(writer_epoch_request())); + let acceptance = submission_acceptance(label); + let receipt = receipt_record(label, WalTickDecision::Applied); + let correlation = correlation_record(label); + let retained_payload = b"retained WSC payload bytes"; + let retained_digest = blake3::hash(retained_payload).into(); + let retained_coordinate = digest("coordinate:wsc-retained-evidence"); + let retained_material = RetainedMaterialRecord { + material_digest: retained_digest, + semantic_coordinate_digest: retained_coordinate, + kind: RetainedMaterialKind::ReadingPayload, + posture: EvidenceMaterialPosture::Present, + }; + let reading_ref = ReadingRefRecord { + reading_id: digest("reading:wsc-retained-evidence"), + semantic_coordinate_digest: retained_coordinate, + payload_digest: retained_digest, + envelope_digest: digest("envelope:wsc-retained-evidence"), + posture: EvidenceMaterialPosture::Present, + }; + + must_ok(store.append_transaction(durable_submission_transaction(label, Lsn::from_raw(0)))); + must_ok(store.append_transaction(durable_tick_transaction( + label, + Lsn::from_raw(2), + WalTickDecision::Applied, + ))); + must_ok(store.seal_segment(epoch_id(), WalSegmentId::from_raw(1))); + let segment_path = store.segment_path(); + let segment_bytes = must_ok(fs::read(&segment_path)); + let last_commit_digest = must_some( + store + .read_commits() + .last() + .map(|commit| commit.commit_digest), + ); + let manifest = WalManifest { + manifest_digest: digest("wsc-retained-evidence:manifest"), + last_committed_lsn: Some(Lsn::from_raw(4)), + last_commit_digest: Some(last_commit_digest), + sealed_segment_count: 1, + }; + must_ok(store.publish_manifest(epoch_id(), manifest)); + + let report = must_ok(recover_filesystem_store(&dir, RecoveryAccessMode::ReadOnly)); + let certificate = build_recovery_certificate( + &report, + None, + 0, + digest("wsc-retained-evidence:frontier"), + digest("wsc-retained-evidence:indexes"), + ); + let writer_epoch = WalWriterEpoch::from_writer_epoch(&writer_epoch); + let projection = project_filesystem_wal_recovery( + &dir, + &report, + std::slice::from_ref(&writer_epoch), + Some(&certificate), + ); + assert_eq!(projection.posture, WalRecoveryProjectionPosture::Present); + let root = must_some(projection.root); + let segment = root.segments[0].clone(); + + let ref_only = must_ok(wsc_ref_only_wal_export( + &root, + wsc_records( + &[retained_material], + &[reading_ref], + &[acceptance], + &[receipt], + &[correlation], + ), + )); + let ref_only_import = must_ok(validate_wsc_ref_only_wal_export(&ref_only, &root)); + assert_eq!(ref_only_import.retention.materials, vec![retained_material]); + assert_eq!(ref_only_import.retention.readings, vec![reading_ref]); + assert_eq!( + ref_only_import.retention.materials[0].semantic_coordinate_digest, + retained_coordinate + ); + assert_eq!( + ref_only_import.retention.materials[0].material_digest, + retained_digest + ); + + let self_contained = must_ok(wsc_self_contained_wal_export( + &root, + &[WscSelfContainedWalSegmentMaterial { + segment_id: segment.segment_id, + segment_bytes: segment_bytes.clone(), + }], + &[WscSelfContainedRetainedMaterial { + material: retained_material, + material_bytes: retained_payload.to_vec(), + }], + wsc_records( + &[retained_material], + &[reading_ref], + &[acceptance], + &[receipt], + &[correlation], + ), + )); + let self_contained_import = must_ok(validate_wsc_self_contained_wal_export( + &self_contained, + &root, + )); + assert_eq!( + self_contained_import.retained_payloads[0].material, + retained_material + ); + assert_eq!( + self_contained_import.retained_payloads[0].material_bytes, + retained_payload + ); + assert_eq!(self_contained_import.retention.readings, vec![reading_ref]); + + let mut cas_store = MemoryTier::new(); + let segment_content_hash = *cas_store.put(&segment_bytes).as_bytes(); + let retained_content_hash = *cas_store.put(retained_payload).as_bytes(); + let cas_addressed = must_ok(wsc_cas_addressed_wal_export( + &root, + &[WscCasAddressedWalSegmentMaterial { + segment_id: segment.segment_id, + content_hash: segment_content_hash, + semantic_coordinate_digest: digest("wsc-retained-evidence:segment"), + byte_len: u64::try_from(segment_bytes.len()).unwrap_or(u64::MAX), + }], + &[WscCasAddressedRetainedMaterialReference { + material_kind: RetainedMaterialKind::ReadingPayload, + content_hash: retained_content_hash, + semantic_coordinate_digest: retained_coordinate, + byte_len: u64::try_from(retained_payload.len()).unwrap_or(u64::MAX), + }], + wsc_records( + &[retained_material], + &[reading_ref], + &[acceptance], + &[receipt], + &[correlation], + ), + )); + let cas_import = must_ok(validate_wsc_cas_addressed_wal_export( + &cas_addressed, + &root, + &EchoCasAvailability(&cas_store), + )); + assert_eq!(cas_import.retention.materials, vec![retained_material]); + assert_eq!(cas_import.retention.readings, vec![reading_ref]); + assert_eq!(cas_import.cas_references.retained_materials.len(), 1); + assert_eq!( + cas_import.cas_references.retained_materials[0].content_hash, + retained_content_hash + ); + + let mut segment_only_cas = MemoryTier::new(); + assert_eq!( + *segment_only_cas.put(&segment_bytes).as_bytes(), + segment_content_hash + ); + let missing = must_err( + validate_wsc_cas_addressed_wal_export( + &cas_addressed, + &root, + &EchoCasAvailability(&segment_only_cas), + ), + "CAS-addressed retained evidence import requires present blobs", + ); + assert!(matches!( + missing, + WscCasAddressedWalImportError::MissingCasBlob { content_hash, .. } + if content_hash == retained_content_hash + )); +} + fn root_with_absolute_locator(root: &WalRoot, path: &str) -> WalRoot { let mut root = root.clone(); root.segments[0].storage_locator = @@ -1461,6 +1640,22 @@ fn correlation_record(label: &str) -> WalReceiptCorrelationRecord { } } +fn wsc_records<'a>( + retained_materials: &'a [RetainedMaterialRecord], + reading_refs: &'a [ReadingRefRecord], + accepted_submissions: &'a [SubmissionAcceptanceRecord], + receipts: &'a [TickReceiptRecord], + correlations: &'a [WalReceiptCorrelationRecord], +) -> WscWalCausalHistoryRecords<'a> { + WscWalCausalHistoryRecords { + retained_materials, + reading_refs, + accepted_submissions, + receipts, + correlations, + } +} + fn retained_material( label: &str, kind: RetainedMaterialKind, diff --git a/docs/workflows.md b/docs/workflows.md index ae90a6e6..31d76c77 100644 --- a/docs/workflows.md +++ b/docs/workflows.md @@ -100,6 +100,20 @@ Use `make verify-pr` or CI for broader package, clippy, rustdoc, and workspace coverage. Pre-push is intentionally the narrowest local proof that the changed slice still compiles and runs. +Local `full` verification keeps broad Cargo test suites CI-owned by default. +It still runs formatting, selected clippy/check lanes, docs/schema lint, and +guards locally, but skips the broad Cargo test lanes unless you explicitly opt +in: + +```sh +VERIFY_LOCAL_FULL_TESTS=1 scripts/verify-local.sh full +``` + +That opt-in is for maintainers who want to pay the local cost. The default +path leaves workspace, `warp-core`, feature-matrix, musl, DIND, rustdoc, and +hook-regression breadth to GitHub Actions while local iteration uses exact +test targets or repo-owned slices. + Checkpoint checks: ```sh diff --git a/scripts/verify-local.sh b/scripts/verify-local.sh index 124d3a90..cd55ea2f 100755 --- a/scripts/verify-local.sh +++ b/scripts/verify-local.sh @@ -25,6 +25,7 @@ VERIFY_USE_NEXTEST="${VERIFY_USE_NEXTEST:-0}" VERIFY_LANE_MODE="${VERIFY_LANE_MODE:-parallel}" VERIFY_LANE_ROOT="${VERIFY_LANE_ROOT:-target/verify-lanes}" VERIFY_TIMING_FILE="${VERIFY_TIMING_FILE:-$STAMP_DIR/timing.jsonl}" +VERIFY_LOCAL_FULL_TESTS="${VERIFY_LOCAL_FULL_TESTS:-0}" VERIFY_LOCAL_RUSTDOC="${VERIFY_LOCAL_RUSTDOC:-0}" VERIFY_LOCAL_HOOK_TESTS="${VERIFY_LOCAL_HOOK_TESTS:-0}" VERIFY_RUN_CACHE_STATE="${VERIFY_RUN_CACHE_STATE:-fresh}" @@ -414,6 +415,10 @@ local_hook_tests_enabled() { [[ "$VERIFY_LOCAL_HOOK_TESTS" == "1" ]] } +local_full_tests_enabled() { + [[ "$VERIFY_LOCAL_FULL_TESTS" == "1" ]] +} + list_changed_branch_files() { if git rev-parse --verify '@{upstream}' >/dev/null 2>&1; then git diff --name-only --diff-filter=ACMRTUXBD '@{upstream}...HEAD' @@ -1671,6 +1676,10 @@ run_full_lane_tests_warp_core() { done } +run_full_tests_ci_owned_notice() { + echo "[verify-local][tests] CI-owned by default; set VERIFY_LOCAL_FULL_TESTS=1 to opt in" +} + run_full_lane_rustdoc() { if ! local_rustdoc_enabled; then echo "[verify-local][rustdoc] CI-owned by default; set VERIFY_LOCAL_RUSTDOC=1 to opt in" @@ -1811,9 +1820,13 @@ run_full_checks_sequential() { run_timed_step "clippy-core" run_full_lane_clippy_core run_timed_step "clippy-support" run_full_lane_clippy_support run_timed_step "clippy-bins" run_full_lane_clippy_bins - run_timed_step "tests-support" run_full_lane_tests_support - run_timed_step "tests-runtime" run_full_lane_tests_runtime - run_timed_step "tests-warp-core" run_full_lane_tests_warp_core + if local_full_tests_enabled; then + run_timed_step "tests-support" run_full_lane_tests_support + run_timed_step "tests-runtime" run_full_lane_tests_runtime + run_timed_step "tests-warp-core" run_full_lane_tests_warp_core + else + run_timed_step "tests-ci-owned" run_full_tests_ci_owned_notice + fi if local_rustdoc_enabled; then run_timed_step "rustdoc" run_full_lane_rustdoc fi @@ -1837,14 +1850,18 @@ run_full_checks_parallel() { if [[ ${#FULL_SCOPE_CLIPPY_BIN_ONLY_PACKAGES[@]} -gt 0 ]]; then lanes+=("clippy-bins" run_full_lane_clippy_bins) fi - if [[ ${#FULL_SCOPE_TEST_SUPPORT_PACKAGES[@]} -gt 0 ]]; then - lanes+=("tests-support" run_full_lane_tests_support) - fi - if [[ "$FULL_SCOPE_WARP_WASM_TEST_MODE" != "none" || "$FULL_SCOPE_ECHO_WASM_ABI_RUN_LIB" == "1" || ${#FULL_SCOPE_ECHO_WASM_ABI_EXTRA_TESTS[@]} -gt 0 ]]; then - lanes+=("tests-runtime" run_full_lane_tests_runtime) - fi - if [[ "$FULL_SCOPE_RUN_WARP_CORE_SMOKE" == "1" ]]; then - lanes+=("tests-warp-core" run_full_lane_tests_warp_core) + if local_full_tests_enabled; then + if [[ ${#FULL_SCOPE_TEST_SUPPORT_PACKAGES[@]} -gt 0 ]]; then + lanes+=("tests-support" run_full_lane_tests_support) + fi + if [[ "$FULL_SCOPE_WARP_WASM_TEST_MODE" != "none" || "$FULL_SCOPE_ECHO_WASM_ABI_RUN_LIB" == "1" || ${#FULL_SCOPE_ECHO_WASM_ABI_EXTRA_TESTS[@]} -gt 0 ]]; then + lanes+=("tests-runtime" run_full_lane_tests_runtime) + fi + if [[ "$FULL_SCOPE_RUN_WARP_CORE_SMOKE" == "1" ]]; then + lanes+=("tests-warp-core" run_full_lane_tests_warp_core) + fi + else + lanes+=("tests-ci-owned" run_full_tests_ci_owned_notice) fi if local_rustdoc_enabled && [[ ${#FULL_SCOPE_RUSTDOC_PACKAGES[@]} -gt 0 ]]; then lanes+=("rustdoc" run_full_lane_rustdoc) diff --git a/tests/hooks/test_verify_local.sh b/tests/hooks/test_verify_local.sh index cdc5c3eb..07294234 100755 --- a/tests/hooks/test_verify_local.sh +++ b/tests/hooks/test_verify_local.sh @@ -444,6 +444,9 @@ EOF if [[ -n "${VERIFY_LOCAL_HOOK_TESTS+x}" ]]; then verify_env+=("VERIFY_LOCAL_HOOK_TESTS=$VERIFY_LOCAL_HOOK_TESTS") fi + if [[ -n "${VERIFY_LOCAL_FULL_TESTS+x}" ]]; then + verify_env+=("VERIFY_LOCAL_FULL_TESTS=$VERIFY_LOCAL_FULL_TESTS") + fi output="$( cd "$tmp" && \ env "${verify_env[@]}" \ @@ -1319,11 +1322,26 @@ else fail "full verification should route clippy through an isolated target dir" printf '%s\n' "$fake_full_output" fi -if printf '%s\n' "$fake_full_output" | grep -q 'target/verify-lanes/full-tests-warp-core'; then - pass "full verification isolates warp-core tests into their own target dir" +if grep -Eq '\[verify-local\] full: lanes=.*tests-ci-owned' <<< "$fake_full_output"; then + pass "full local verification leaves broad Cargo test lanes to CI by default" else - fail "full verification should route warp-core tests through an isolated target dir" + fail "full local verification should show the CI-owned Cargo test lane" + printf '%s\n' "$fake_full_output" +fi +if printf '%s\n' "$fake_full_output" | grep -q 'target/verify-lanes/full-tests-warp-core'; then + fail "full local verification should not launch broad Cargo test lanes by default" printf '%s\n' "$fake_full_output" +else + pass "full local verification avoids broad Cargo test lanes by default" +fi +VERIFY_LOCAL_FULL_TESTS=1 +fake_full_with_tests_output="$(run_fake_verify full crates/warp-core/src/lib.rs)" +unset VERIFY_LOCAL_FULL_TESTS +if printf '%s\n' "$fake_full_with_tests_output" | grep -q 'target/verify-lanes/full-tests-warp-core'; then + pass "full local verification keeps an explicit Cargo test lane opt-in" +else + fail "full local verification should keep an explicit Cargo test lane opt-in" + printf '%s\n' "$fake_full_with_tests_output" fi if printf '%s\n' "$fake_full_output" | grep -q 'doc -p warp-core'; then fail "full local verification should leave rustdoc to CI by default" @@ -1644,7 +1662,9 @@ else printf '%s\n' "$fake_pre_push_warp_math_bin_src_output" fi +VERIFY_LOCAL_FULL_TESTS=1 fake_warp_core_default_output="$(run_fake_verify full crates/warp-core/src/provenance_store.rs)" +unset VERIFY_LOCAL_FULL_TESTS if printf '%s\n' "$fake_warp_core_default_output" | grep -q 'test -p warp-core --lib'; then pass "warp-core default smoke keeps the lib test lane" else @@ -1658,7 +1678,9 @@ else pass "warp-core default smoke avoids inbox when the file family does not need it" fi +VERIFY_LOCAL_FULL_TESTS=1 fake_warp_core_optic_artifact_output="$(run_fake_verify full crates/warp-core/src/optic_artifact.rs)" +unset VERIFY_LOCAL_FULL_TESTS if printf '%s\n' "$fake_warp_core_optic_artifact_output" | grep -q -- '--test optic_artifact_registry_tests'; then pass "optic artifact changes pull the registry smoke test" else @@ -1684,7 +1706,9 @@ else printf '%s\n' "$fake_warp_core_optic_artifact_output" fi +VERIFY_LOCAL_FULL_TESTS=1 fake_warp_core_causal_facts_output="$(run_fake_verify full crates/warp-core/src/causal_facts.rs)" +unset VERIFY_LOCAL_FULL_TESTS if printf '%s\n' "$fake_warp_core_causal_facts_output" | grep -q -- '--test causal_fact_publication_tests'; then pass "causal fact source changes pull the causal fact publication smoke test" else @@ -1704,7 +1728,9 @@ else printf '%s\n' "$fake_warp_core_causal_facts_output" fi +VERIFY_LOCAL_FULL_TESTS=1 fake_warp_core_runtime_output="$(run_fake_verify full crates/warp-core/src/coordinator.rs)" +unset VERIFY_LOCAL_FULL_TESTS if printf '%s\n' "$fake_warp_core_runtime_output" | grep -q -- '--test inbox'; then pass "runtime-facing warp-core changes pull the inbox smoke test" else @@ -1712,7 +1738,9 @@ else printf '%s\n' "$fake_warp_core_runtime_output" fi +VERIFY_LOCAL_FULL_TESTS=1 fake_warp_core_playback_output="$(run_fake_verify full crates/warp-core/src/playback.rs)" +unset VERIFY_LOCAL_FULL_TESTS if printf '%s\n' "$fake_warp_core_playback_output" | grep -q -- '--test playback_cursor_tests'; then pass "playback changes pull the playback cursor smoke test" else @@ -1726,7 +1754,9 @@ else printf '%s\n' "$fake_warp_core_playback_output" fi +VERIFY_LOCAL_FULL_TESTS=1 fake_warp_math_prng_output="$(run_fake_verify full crates/warp-math/src/prng.rs)" +unset VERIFY_LOCAL_FULL_TESTS if printf '%s\n' "$fake_warp_math_prng_output" | grep -q -- 'test -p warp-math --features golden_prng --test prng_golden_regression'; then pass "PRNG changes pull the golden regression smoke test" else @@ -1734,7 +1764,9 @@ else printf '%s\n' "$fake_warp_math_prng_output" fi +VERIFY_LOCAL_FULL_TESTS=1 fake_warp_wasm_lib_output="$(run_fake_verify full crates/warp-wasm/src/lib.rs)" +unset VERIFY_LOCAL_FULL_TESTS if printf '%s\n' "$fake_warp_wasm_lib_output" | grep -q 'test -p warp-wasm --lib'; then pass "warp-wasm lib changes use the plain lib smoke lane" else @@ -1748,7 +1780,9 @@ else pass "warp-wasm lib changes avoid the engine smoke lane" fi +VERIFY_LOCAL_FULL_TESTS=1 fake_warp_wasm_kernel_output="$(run_fake_verify full crates/warp-wasm/src/warp_kernel.rs)" +unset VERIFY_LOCAL_FULL_TESTS if printf '%s\n' "$fake_warp_wasm_kernel_output" | grep -q -- 'test -p warp-wasm --features engine --lib'; then pass "warp-kernel changes use the engine-enabled lib smoke lane" else @@ -1756,7 +1790,9 @@ else printf '%s\n' "$fake_warp_wasm_kernel_output" fi +VERIFY_LOCAL_FULL_TESTS=1 fake_echo_wasm_abi_kernel_port_output="$(run_fake_verify full crates/echo-wasm-abi/src/kernel_port.rs)" +unset VERIFY_LOCAL_FULL_TESTS if printf '%s\n' "$fake_echo_wasm_abi_kernel_port_output" | grep -q -- 'test -p echo-wasm-abi --lib'; then pass "echo-wasm-abi kernel-port changes keep the lib smoke lane" else @@ -1764,7 +1800,9 @@ else printf '%s\n' "$fake_echo_wasm_abi_kernel_port_output" fi +VERIFY_LOCAL_FULL_TESTS=1 fake_echo_wasm_abi_canonical_output="$(run_fake_verify full crates/echo-wasm-abi/src/canonical.rs)" +unset VERIFY_LOCAL_FULL_TESTS if printf '%s\n' "$fake_echo_wasm_abi_canonical_output" | grep -q -- '--test canonical_vectors'; then pass "canonical ABI changes pull canonical vector coverage" else diff --git a/xtask/src/main.rs b/xtask/src/main.rs index bcaf3a82..12bccbff 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -785,6 +785,14 @@ fn build_test_slice_commands(slice: TestSlice) -> Vec { "causal_wal_tests", "topology_", ]), + cargo_command([ + "test", + "-p", + "warp-core", + "--test", + "causal_wal_tests", + "wsc_retained_evidence_export_modes", + ]), cargo_command([ "test", "-p", @@ -6703,7 +6711,7 @@ mod tests { #[test] fn test_slice_durability_release_stays_explicit() { let commands = build_test_slice_commands(TestSlice::DurabilityRelease); - assert_eq!(commands.len(), 11); + assert_eq!(commands.len(), 12); let expected = [ ( @@ -6787,6 +6795,17 @@ mod tests { "topology_", ], ), + ( + "cargo", + vec![ + "test", + "-p", + "warp-core", + "--test", + "causal_wal_tests", + "wsc_retained_evidence_export_modes", + ], + ), ( "cargo", vec![