diff --git a/CHANGELOG.md b/CHANGELOG.md index f3aaf206..9dc13be1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ ### Added +- Settled external-action candidates can now be reconciled idempotently after + acknowledgement loss without a WAL store, transition context, or claim + grant. An exact retained candidate returns the original admitted settlement + and commit digest without appending history or making adapter execution + reachable. A different valid candidate conflicts, malformed candidates fail + ordinary settlement validation, and duplicated settlement records remain a + recovery obstruction. - Echo now independently admits compiler-produced Edict Core and Target IR for one non-callable external request, verifies its exact source, target profile, result, basis, and capability closure, independently corroborates the complete diff --git a/crates/warp-core/src/external_action.rs b/crates/warp-core/src/external_action.rs index cf60e92a..1095a8a8 100644 --- a/crates/warp-core/src/external_action.rs +++ b/crates/warp-core/src/external_action.rs @@ -1347,6 +1347,42 @@ pub fn admit_external_action_settlement( }) } +/// Reconciles one retained adapter settlement after acknowledgement loss. +/// +/// This path exposes no WAL store, transition context, or claim grant. It can +/// therefore return only the exact settlement that is already durable. A +/// different valid candidate is a conflict; a malformed candidate fails the +/// ordinary request-and-claim validation before comparison. +pub fn reconcile_external_action_settlement_retry( + coordinator: &ExternalActionCoordinatorV1, + candidate: ExternalActionSettlementCandidateV1, +) -> Result { + coordinator.ensure_ready()?; + let recovered = coordinator + .index + .get(candidate.request_id) + .ok_or(ExternalActionProtocolErrorV1::MissingRequest)?; + let claim = recovered + .claim + .ok_or(ExternalActionProtocolErrorV1::MissingClaim)?; + validate_settlement_candidate(&recovered.request, &claim, &candidate)?; + let candidate = ExternalActionSettlementV1::from_candidate(candidate); + let settlement = recovered + .settlement + .as_ref() + .ok_or(ExternalActionProtocolErrorV1::MissingSettlement)?; + let settlement_commit_digest = recovered + .settlement_commit_digest + .ok_or(ExternalActionProtocolErrorV1::MissingSettlement)?; + if settlement != &candidate { + return Err(ExternalActionProtocolErrorV1::ConflictingSettlement); + } + Ok(AdmittedExternalActionSettlementV1 { + settlement: settlement.clone(), + settlement_commit_digest, + }) +} + /// Observes request, claim, and settlement posture in an arbitrary recovery report. /// /// This projection carries no transition or replay authority. Use @@ -1438,7 +1474,10 @@ fn apply_recovered_settlement( .ok_or(ExternalActionProtocolErrorV1::MissingClaim)?; validate_settlement(&entry.request, &claim, &settlement)?; if let Some(existing) = &entry.settlement { - return if existing == &settlement { + let existing_commit = entry + .settlement_commit_digest + .ok_or(ExternalActionProtocolErrorV1::MissingSettlement)?; + return if existing == &settlement && existing_commit == commit_digest { Err(ExternalActionProtocolErrorV1::DuplicateSettlement) } else { Err(ExternalActionProtocolErrorV1::ConflictingSettlement) @@ -1782,8 +1821,11 @@ mod tests { } } - #[test] - fn conflicting_recovered_settlement_is_obstructed() { + fn claimed_index() -> ( + ExternalActionRequestV1, + ExternalActionClaimV1, + RecoveredExternalActionIndexV1, + ) { let request = request(); let claim = ExternalActionClaimV1::for_request( &request, @@ -1802,37 +1844,105 @@ mod tests { settlement_commit_digest: None, posture: RecoveredExternalActionPostureV1::Claimed, })); - let first = - ExternalActionSettlementV1::from_candidate(ExternalActionSettlementCandidateV1::new( - request.request_id, - claim.attempt_id, - claim.adapter_id, - ExternalActionSettlementKindV1::Succeeded, - request.settlement_schema_digest, - request.basis_digest, - b"first".to_vec(), - digest("test.schema-evidence"), - digest("test.external-evidence"), - )); + (request, claim, index) + } + + fn settlement( + request: &ExternalActionRequestV1, + claim: &ExternalActionClaimV1, + bytes: &[u8], + ) -> ExternalActionSettlementV1 { + ExternalActionSettlementV1::from_candidate(ExternalActionSettlementCandidateV1::new( + request.request_id, + claim.attempt_id, + claim.adapter_id, + ExternalActionSettlementKindV1::Succeeded, + request.settlement_schema_digest, + request.basis_digest, + bytes.to_vec(), + digest("test.schema-evidence"), + digest("test.external-evidence"), + )) + } + + #[test] + fn identical_recovered_settlement_is_a_duplicate() { + let (request, claim, mut index) = claimed_index(); + let settlement = settlement(&request, &claim, b"same"); + let commit = digest("settlement.commit"); + + assert_eq!( + apply_recovered_settlement(&mut index, settlement.clone(), commit), + Ok(()) + ); + let root = index.root_digest(); + assert_eq!( + apply_recovered_settlement(&mut index, settlement, commit), + Err(ExternalActionProtocolErrorV1::DuplicateSettlement) + ); + assert_eq!(index.len(), 1); + assert_eq!(index.root_digest(), root); + } + + #[test] + fn conflicting_recovered_settlement_is_obstructed() { + let (request, claim, mut index) = claimed_index(); + let first = settlement(&request, &claim, b"first"); assert_eq!( apply_recovered_settlement(&mut index, first, digest("settlement.commit")), Ok(()) ); - let conflicting = - ExternalActionSettlementV1::from_candidate(ExternalActionSettlementCandidateV1::new( - request.request_id, - claim.attempt_id, - claim.adapter_id, - ExternalActionSettlementKindV1::Succeeded, - request.settlement_schema_digest, - request.basis_digest, - b"second".to_vec(), - digest("test.schema-evidence"), - digest("test.external-evidence"), - )); + let conflicting = settlement(&request, &claim, b"second"); assert_eq!( apply_recovered_settlement(&mut index, conflicting, digest("conflict.commit")), Err(ExternalActionProtocolErrorV1::ConflictingSettlement) ); } + + #[test] + fn identical_settlement_under_another_commit_is_conflicting() { + let (request, claim, mut index) = claimed_index(); + let settlement = settlement(&request, &claim, b"same"); + assert_eq!( + apply_recovered_settlement(&mut index, settlement.clone(), digest("settlement.commit")), + Ok(()) + ); + assert_eq!( + apply_recovered_settlement( + &mut index, + settlement, + digest("different-settlement.commit") + ), + Err(ExternalActionProtocolErrorV1::ConflictingSettlement) + ); + } + + #[test] + fn fixed_seed_settlement_mutations_are_conflicting() { + const SEED: u64 = 0x5e77_1e5e_77e5_0001; + let (request, claim, mut index) = claimed_index(); + let first = settlement(&request, &claim, &SEED.to_le_bytes()); + assert_eq!( + apply_recovered_settlement(&mut index, first, digest("property-settlement.commit")), + Ok(()) + ); + + let mut state = SEED; + for ordinal in 0_u8..32 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + let mut bytes = state.to_le_bytes().to_vec(); + bytes.push(ordinal); + let conflicting = settlement(&request, &claim, &bytes); + assert_eq!( + apply_recovered_settlement( + &mut index, + conflicting, + digest("property-conflict.commit") + ), + Err(ExternalActionProtocolErrorV1::ConflictingSettlement) + ); + } + } } diff --git a/crates/warp-core/tests/external_action_protocol_tests.rs b/crates/warp-core/tests/external_action_protocol_tests.rs index fd3c5d9e..8dda756c 100644 --- a/crates/warp-core/tests/external_action_protocol_tests.rs +++ b/crates/warp-core/tests/external_action_protocol_tests.rs @@ -19,12 +19,13 @@ use warp_core::causal_wal::{ }; use warp_core::external_action::{ admit_external_action_settlement, claim_external_action, observe_external_actions, - record_external_action_request, ExternalActionAdapterAuthorizationV1, - ExternalActionAdapterBindingV1, ExternalActionAdapterIdV1, ExternalActionAdapterRegistryV1, - ExternalActionBudgetV1, ExternalActionClaimGrantV1, ExternalActionCoordinatorV1, - ExternalActionOperationIdV1, ExternalActionProtocolErrorV1, ExternalActionRequestV1, - ExternalActionSettlementCandidateV1, ExternalActionSettlementKindV1, - ExternalActionTransactionContextV1, RecoveredExternalActionPostureV1, + reconcile_external_action_settlement_retry, record_external_action_request, + ExternalActionAdapterAuthorizationV1, ExternalActionAdapterBindingV1, + ExternalActionAdapterIdV1, ExternalActionAdapterRegistryV1, ExternalActionBudgetV1, + ExternalActionClaimGrantV1, ExternalActionCoordinatorV1, ExternalActionOperationIdV1, + ExternalActionProtocolErrorV1, ExternalActionRequestV1, ExternalActionSettlementCandidateV1, + ExternalActionSettlementKindV1, ExternalActionTransactionContextV1, + RecoveredExternalActionPostureV1, }; use warp_core::{Hash, WorldlineId}; @@ -796,7 +797,7 @@ fn filesystem_reopen_recovers_settlement_without_adapter_reexecution() { let wal_dir = TempWalDir::new("reopen"); let request = request_with("filesystem-reopen", 21, 128); let result_bytes = b"durable observed bytes".to_vec(); - { + let (retry, admitted) = { let mut store = must_ok(FilesystemWalStore::open( &wal_dir.0, WalSegmentId::from_raw(1), @@ -839,7 +840,7 @@ fn filesystem_reopen_recovers_settlement_without_adapter_reexecution() { ExternalActionSettlementKindV1::Succeeded, result_bytes.clone(), ); - must_ok(admit_external_action_settlement( + let admitted = must_ok(admit_external_action_settlement( &mut store, &mut coordinator, context_with_durability( @@ -847,9 +848,10 @@ fn filesystem_reopen_recovers_settlement_without_adapter_reexecution() { WalDurabilityMode::StrictFilesystem, ), grant, - candidate, + candidate.clone(), )); - } + (candidate, admitted) + }; let report = must_ok(recover_filesystem_store( &wal_dir.0, @@ -872,6 +874,21 @@ fn filesystem_reopen_recovers_settlement_without_adapter_reexecution() { .map(|settlement| settlement.canonical_result_bytes.as_slice()), Some(result_bytes.as_slice()) ); + + let reopened = must_ok(FilesystemWalStore::open( + &wal_dir.0, + WalSegmentId::from_raw(1), + )); + let commits_before = reopened.read_commits().len(); + let recovered_coordinator = must_ok(ExternalActionCoordinatorV1::recover(&reopened)); + assert_eq!( + must_ok(reconcile_external_action_settlement_retry( + &recovered_coordinator, + retry + )), + admitted + ); + assert_eq!(reopened.read_commits().len(), commits_before); } #[test] @@ -1043,6 +1060,215 @@ fn duplicate_settlement_is_a_recovery_obstruction() { ); } +#[test] +fn identical_settlement_retry_returns_the_original_fact_without_wal_growth() { + let mut store = store(); + let mut coordinator = coordinator(&store); + let request = request_with("retry-identical", 66, 128); + let recorded = record( + &mut store, + &mut coordinator, + request, + "request:retry-identical", + ); + let grant = claim( + &mut store, + &mut coordinator, + recorded, + "claim:retry-identical", + ); + let retry = candidate( + &grant, + ExternalActionSettlementKindV1::Succeeded, + b"retained".to_vec(), + ); + let admitted = must_ok(admit_external_action_settlement( + &mut store, + &mut coordinator, + context("settlement:retry-identical"), + grant, + retry.clone(), + )); + let frames_before = store.read_frames().len(); + let commits_before = store.read_commits().len(); + let root_before = coordinator.observed_index().root_digest(); + + for _ in 0..64 { + let retried = must_ok(reconcile_external_action_settlement_retry( + &coordinator, + retry.clone(), + )); + assert_eq!(retried, admitted); + } + assert_eq!(store.read_frames().len(), frames_before); + assert_eq!(store.read_commits().len(), commits_before); + assert_eq!(coordinator.observed_index().root_digest(), root_before); +} + +#[test] +fn conflicting_settlement_retry_obstructs_without_wal_growth() { + let mut store = store(); + let mut coordinator = coordinator(&store); + let request = request_with("retry-conflict", 67, 128); + let recorded = record( + &mut store, + &mut coordinator, + request, + "request:retry-conflict", + ); + let grant = claim( + &mut store, + &mut coordinator, + recorded, + "claim:retry-conflict", + ); + let first = candidate( + &grant, + ExternalActionSettlementKindV1::Succeeded, + b"first".to_vec(), + ); + let conflicting = candidate( + &grant, + ExternalActionSettlementKindV1::Succeeded, + b"conflicting".to_vec(), + ); + let kind_conflicting = candidate( + &grant, + ExternalActionSettlementKindV1::Failed, + b"first".to_vec(), + ); + must_ok(admit_external_action_settlement( + &mut store, + &mut coordinator, + context("settlement:retry-conflict"), + grant, + first, + )); + let frames_before = store.read_frames().len(); + let commits_before = store.read_commits().len(); + + assert_eq!( + reconcile_external_action_settlement_retry(&coordinator, conflicting), + Err(ExternalActionProtocolErrorV1::ConflictingSettlement) + ); + assert_eq!( + reconcile_external_action_settlement_retry(&coordinator, kind_conflicting), + Err(ExternalActionProtocolErrorV1::ConflictingSettlement) + ); + assert_eq!(store.read_frames().len(), frames_before); + assert_eq!(store.read_commits().len(), commits_before); +} + +#[test] +fn malformed_settlement_retry_fails_validation_before_comparison() { + let mut store = store(); + let mut coordinator = coordinator(&store); + let request = request_with("retry-malformed", 68, 128); + let recorded = record( + &mut store, + &mut coordinator, + request, + "request:retry-malformed", + ); + let grant = claim( + &mut store, + &mut coordinator, + recorded, + "claim:retry-malformed", + ); + let first = candidate( + &grant, + ExternalActionSettlementKindV1::Succeeded, + b"first".to_vec(), + ); + let mut malformed = first.clone(); + malformed.declared_result_digest = digest("malformed-result-digest"); + must_ok(admit_external_action_settlement( + &mut store, + &mut coordinator, + context("settlement:retry-malformed"), + grant, + first, + )); + let commits_before = store.read_commits().len(); + + assert_eq!( + reconcile_external_action_settlement_retry(&coordinator, malformed), + Err(ExternalActionProtocolErrorV1::SettlementResultDigestMismatch) + ); + assert_eq!(store.read_commits().len(), commits_before); +} + +#[test] +fn settlement_retry_cannot_create_the_initial_settlement() { + let mut store = store(); + let mut coordinator = coordinator(&store); + let request = request_with("retry-before-settlement", 70, 128); + let recorded = record( + &mut store, + &mut coordinator, + request, + "request:retry-before-settlement", + ); + let grant = claim( + &mut store, + &mut coordinator, + recorded, + "claim:retry-before-settlement", + ); + let retry = candidate( + &grant, + ExternalActionSettlementKindV1::Succeeded, + b"not-yet-admitted".to_vec(), + ); + let commits_before = store.read_commits().len(); + + assert_eq!( + reconcile_external_action_settlement_retry(&coordinator, retry), + Err(ExternalActionProtocolErrorV1::MissingSettlement) + ); + assert_eq!(store.read_commits().len(), commits_before); +} + +#[test] +fn recovered_settlement_retry_remains_idempotent() { + let mut store = store(); + let mut coordinator = coordinator(&store); + let request = request_with("retry-recovered", 69, 128); + let recorded = record( + &mut store, + &mut coordinator, + request, + "request:retry-recovered", + ); + let grant = claim( + &mut store, + &mut coordinator, + recorded, + "claim:retry-recovered", + ); + let retry = candidate( + &grant, + ExternalActionSettlementKindV1::Succeeded, + b"recovered".to_vec(), + ); + let admitted = must_ok(admit_external_action_settlement( + &mut store, + &mut coordinator, + context("settlement:retry-recovered"), + grant, + retry.clone(), + )); + let recovered = must_ok(ExternalActionCoordinatorV1::recover(&store)); + + assert_eq!( + must_ok(reconcile_external_action_settlement_retry( + &recovered, retry + )), + admitted + ); +} + #[derive(Debug)] struct SnapshotCountingStore { inner: InMemoryWalStore, diff --git a/docs/adr/0026-durable-external-action-settlement.md b/docs/adr/0026-durable-external-action-settlement.md index 7e5ec96a..43595cc3 100644 --- a/docs/adr/0026-durable-external-action-settlement.md +++ b/docs/adr/0026-durable-external-action-settlement.md @@ -98,8 +98,17 @@ A settlement binds the exact request, attempt, adapter, basis, settlement schema, canonical result bytes, result digest, schema-admission evidence, and external evidence. Echo rejects mismatched claims, stale bases, wrong schemas, missing schema-admission or external evidence, digest substitution, oversize -results, duplicate settlements, conflicting settlements, malformed payloads, -and unknown outcome codes. +results, duplicate settlement records, conflicting settlements, malformed +payloads, and unknown outcome codes. + +An adapter that retained its candidate across an acknowledgement loss may +reconcile it against already-settled history. The reconciliation surface +receives no WAL store, transaction context, or claim grant. It validates the +candidate against the recorded request and claim, returns the original +settlement and original commit digest when every field is exact, and appends no +history. A different valid candidate is `ConflictingSettlement`; a malformed +candidate fails ordinary settlement validation before comparison. This is +idempotent result delivery, not permission to execute the adapter again. The schema-admission evidence is a protocol binding, not a general schema engine. Each operation profile must define which validator produces that @@ -166,6 +175,10 @@ Replay does not invoke an adapter. Consulting the current external world again requires a new program transition and a new request; changing worldline or basis changes request identity. +The idempotent retry path does not make duplicate WAL history admissible. An +identical candidate is reconciled before any second transaction exists; an +already duplicated settlement record remains a recovery obstruction. + An arbitrary `RecoveryScanReport` produces observation-only lifecycle values. It cannot mint request-transition tokens, adapter work grants, or resumable settlement facts. `ExternalActionCoordinatorV1::recover` alone reads one diff --git a/docs/topics/ExternalActions.md b/docs/topics/ExternalActions.md index 81133f8f..e0c5707b 100644 --- a/docs/topics/ExternalActions.md +++ b/docs/topics/ExternalActions.md @@ -107,6 +107,15 @@ Before generic WAL admission, the operation profile independently validates: Malformed or substituted candidates fail before the settlement transaction. +If an adapter loses the acknowledgement after settlement commit, it may submit +the retained candidate to +`reconcile_external_action_settlement_retry`. That function has no store, +transaction context, or claim grant. It returns the exact already-admitted +settlement and its original commit digest without appending history. A +different valid candidate conflicts, while a malformed candidate fails the +same generic validation used for first admission. Reconciliation never makes +adapter execution reachable. + ## Recovery And Replay Recovery reconstructs `Requested`, `Claimed`, or the exact settled outcome @@ -117,6 +126,10 @@ Settled replay returns the canonical bytes retained in the WAL. It does not open the capability directory again. Removing or mutating source files after settlement therefore cannot change replay. +Duplicate settlement records still obstruct recovery. Idempotency applies to a +retained candidate reconciled before another WAL transaction, not to duplicated +history. + ## Evidence - `crates/warp-core/src/external_action_adapter.rs`