From 9e60e3f9278831494eb35364e9b9bb05dc720b89 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:02:21 +0500 Subject: [PATCH 01/62] fix(zk): bind C0 proofs to canonical request context --- crates/zk-helpers/src/circuits/commitments.rs | 84 ++++++++++ .../src/actors/proof_verification.rs | 102 ++++++++++-- .../src/domain/proof_verification.rs | 157 ++++++++++++++++-- 3 files changed, 316 insertions(+), 27 deletions(-) diff --git a/crates/zk-helpers/src/circuits/commitments.rs b/crates/zk-helpers/src/circuits/commitments.rs index 2321224d8d..2210af499c 100644 --- a/crates/zk-helpers/src/circuits/commitments.rs +++ b/crates/zk-helpers/src/circuits/commitments.rs @@ -15,7 +15,10 @@ use crate::utils::compute_safe; use ark_bn254::Fr as Field; use ark_ff::BigInteger; use ark_ff::PrimeField; +use e3_fhe_params::{build_pair_for_preset, BfvPreset}; use e3_polynomial::{CrtPolynomial, Polynomial}; +use fhe::bfv::PublicKey; +use fhe_traits::DeserializeParametrized; use num_bigint::BigInt; use std::slice::from_ref; @@ -197,6 +200,50 @@ pub fn compute_dkg_pk_commitment(pk0: &CrtPolynomial, pk1: &CrtPolynomial, bit_p BigInt::from_bytes_le(num_bigint::Sign::Plus, &commitment_bytes) } +/// Compute the C0 `pk_commitment` for a serialized DKG BFV public key. +/// +/// The C0 proof attests to the two polynomials inside the advertised public key. Network +/// consumers must compare this value with the proof's public output before accepting the key; +/// verifying a proof without that comparison would allow the proof for one key to be attached to +/// different key bytes. +pub fn compute_dkg_pk_commitment_from_public_key_bytes( + public_key_bytes: &[u8], + preset: BfvPreset, +) -> Result<[u8; 32], crate::CircuitsErrors> { + // `build_pair_for_preset` accepts the threshold member of a preset pair. Callers may already + // have resolved the DKG counterpart, so normalize either representation here. + let threshold_preset = preset.threshold_counterpart().unwrap_or(preset); + let (_, dkg_params) = build_pair_for_preset(threshold_preset) + .map_err(|e| crate::CircuitsErrors::Other(format!("BFV preset pair: {e}")))?; + let public_key = PublicKey::from_bytes(public_key_bytes, &dkg_params) + .map_err(|e| crate::CircuitsErrors::Other(format!("PublicKey deserialize: {e:?}")))?; + + let pk0 = public_key + .c + .first() + .ok_or_else(|| crate::CircuitsErrors::Other("PublicKey is missing pk0".to_string()))?; + let pk1 = public_key + .c + .get(1) + .ok_or_else(|| crate::CircuitsErrors::Other("PublicKey is missing pk1".to_string()))?; + let moduli = dkg_params.moduli(); + let pk0 = crate::math::fhe_poly_to_crt_centered(pk0, moduli) + .map_err(|e| crate::CircuitsErrors::Other(format!("pk0 conversion: {e}")))?; + let pk1 = crate::math::fhe_poly_to_crt_centered(pk1, moduli) + .map_err(|e| crate::CircuitsErrors::Other(format!("pk1 conversion: {e}")))?; + let commitment = compute_dkg_pk_commitment(&pk0, &pk1, crate::compute_modulus_bit(&dkg_params)); + + let (_, be_bytes) = commitment.to_bytes_be(); + if be_bytes.len() > 32 { + return Err(crate::CircuitsErrors::Other( + "C0 pk_commitment does not fit in a field element".to_string(), + )); + } + let mut padded = [0u8; 32]; + padded[32 - be_bytes.len()..].copy_from_slice(&be_bytes); + Ok(padded) +} + /// Compute a commitment to the threshold public key by flattening pk0 and hashing. /// /// This matches the Noir `compute_threshold_pk_commitment` function exactly, @@ -709,4 +756,41 @@ mod tests { assert_eq!(commitment, expected_padded); } + + #[test] + fn compute_dkg_pk_commitment_from_public_key_bytes_matches_circuit_inputs() { + use crate::circuits::dkg::pk::{Bits, Inputs, PkCircuitData}; + use crate::Computation; + use e3_fhe_params::{build_pair_for_preset, BfvPreset}; + use fhe::bfv::{PublicKey, SecretKey}; + use fhe_traits::Serialize; + + let preset = BfvPreset::InsecureThreshold512; + let (_, dkg_params) = build_pair_for_preset(preset).unwrap(); + let mut rng = rand::rng(); + let secret_key = SecretKey::random(&dkg_params, &mut rng); + let public_key = PublicKey::new(&secret_key, &mut rng); + let public_key_bytes = public_key.to_bytes(); + + let actual = + compute_dkg_pk_commitment_from_public_key_bytes(&public_key_bytes, preset).unwrap(); + + let inputs = Inputs::compute(preset, &PkCircuitData { public_key }).unwrap(); + let bits = Bits::compute(preset, &()).unwrap(); + let expected = compute_dkg_pk_commitment(&inputs.pk0is, &inputs.pk1is, bits.pk_bit); + let (_, expected_bytes) = expected.to_bytes_be(); + let mut expected_padded = [0u8; 32]; + expected_padded[32 - expected_bytes.len()..].copy_from_slice(&expected_bytes); + + assert_eq!(actual, expected_padded); + } + + #[test] + fn compute_dkg_pk_commitment_rejects_malformed_public_key() { + assert!(compute_dkg_pk_commitment_from_public_key_bytes( + b"not a BFV public key", + e3_fhe_params::BfvPreset::InsecureThreshold512, + ) + .is_err()); + } } diff --git a/crates/zk-prover/src/actors/proof_verification.rs b/crates/zk-prover/src/actors/proof_verification.rs index 8ada6277c6..f45cdf2bfd 100644 --- a/crates/zk-prover/src/actors/proof_verification.rs +++ b/crates/zk-prover/src/actors/proof_verification.rs @@ -22,10 +22,10 @@ use e3_events::{ }; use e3_fhe_params::BfvPreset; use e3_utils::NotifySync; -use e3_zk_helpers::CiphernodesCommitteeSize; +use e3_zk_helpers::{compute_dkg_pk_commitment_from_public_key_bytes, CiphernodesCommitteeSize}; use tracing::{error, info, warn}; -use crate::domain::proof_verification::validate_external_key; +use crate::domain::proof_verification::{validate_external_key, validate_external_key_commitment}; #[derive(Debug, Message)] #[rtype(result = "()")] @@ -58,6 +58,9 @@ pub struct ProofVerificationActor { pending: HashMap<(E3id, u64), PendingVerification>, /// Tracks preset + committee per E3 so we can derive `artifacts_dir` for proof verification. presets: HashMap, + /// Canonical finalized committee in party-id order. A C0 signer must own the party slot whose + /// BFV key it advertises; recovering any valid ECDSA address is not sufficient. + committees: HashMap>, } impl ProofVerificationActor { @@ -67,6 +70,7 @@ impl ProofVerificationActor { verifier, pending: HashMap::new(), presets: HashMap::new(), + committees: HashMap::new(), } } @@ -76,7 +80,9 @@ impl ProofVerificationActor { ) -> Addr { let addr = Self::new(bus, verifier).start(); bus.subscribe(EventType::CiphernodeSelected, addr.clone().into()); + bus.subscribe(EventType::CommitteeFinalized, addr.clone().into()); bus.subscribe(EventType::EncryptionKeyReceived, addr.clone().into()); + bus.subscribe(EventType::E3RequestComplete, addr.clone().into()); addr } @@ -86,7 +92,44 @@ impl ProofVerificationActor { ctx: &Context, ) { let (msg, ec) = msg.into_components(); + let pending_key = (msg.e3_id.clone(), msg.key.party_id); + if self.pending.contains_key(&pending_key) { + warn!( + e3_id = %msg.e3_id, + party_id = msg.key.party_id, + "C0 verification is already pending for party — ignoring duplicate" + ); + return; + } + + let Some((preset, committee_size)) = self.presets.get(&msg.e3_id).copied() else { + error!( + "No BfvPreset known for e3_id={} — cannot determine circuit artifacts directory. \ + This can happen if CiphernodeSelected was missed (e.g. after restart). Rejecting key from party {}.", + msg.e3_id, msg.key.party_id + ); + return; + }; + let Some(expected_signer) = self + .committees + .get(&msg.e3_id) + .and_then(|committee| { + usize::try_from(msg.key.party_id) + .ok() + .and_then(|i| committee.get(i)) + }) + .copied() + else { + error!( + e3_id = %msg.e3_id, + party_id = msg.key.party_id, + "No finalized committee member for C0 party slot — rejecting key" + ); + return; + }; let validated = match validate_external_key( + &msg.e3_id, + &expected_signer, msg.key.party_id, msg.key.proof.as_ref(), msg.key.signed_payload.as_ref(), @@ -102,25 +145,35 @@ impl ProofVerificationActor { .proof .clone() .expect("proof present after validation"); + let key_commitment = + match compute_dkg_pk_commitment_from_public_key_bytes(&msg.key.pk_bfv, preset) { + Ok(commitment) => commitment, + Err(err) => { + error!( + e3_id = %msg.e3_id, + party_id = msg.key.party_id, + "Could not bind C0 proof to advertised BFV key: {err} — rejecting key" + ); + return; + } + }; + if let Err(reason) = + validate_external_key_commitment(msg.key.party_id, &proof, &key_commitment) + { + error!("{reason}"); + return; + } // Store the signed payload so we can reference it in the verification response self.pending.insert( - (msg.e3_id.clone(), msg.key.party_id), + pending_key, PendingVerification { signed_payload: validated.signed_payload, recovered_signer: validated.recovered_signer, }, ); - let Some((preset, committee)) = self.presets.get(&msg.e3_id).copied() else { - error!( - "No BfvPreset known for e3_id={} — cannot determine circuit artifacts directory. \ - This can happen if CiphernodeSelected was missed (e.g. after restart). Rejecting key from party {}.", - msg.e3_id, msg.key.party_id - ); - return; - }; - let artifacts_dir = preset.artifacts_dir_for_committee(committee.as_str()); + let artifacts_dir = preset.artifacts_dir_for_committee(committee_size.as_str()); let request = TypedEvent::new( ZkVerificationRequest { @@ -181,9 +234,34 @@ impl Handler for ProofVerificationActor { } } } + InterfoldEventData::CommitteeFinalized(mut data) => { + // The EVM decoder already emits canonical address order, but sorting again keeps + // this trust boundary correct for replayed/test-produced events as well. + data.sort_by_score(); + let parsed: Result, _> = + data.committee.iter().map(|node| node.parse()).collect(); + match parsed { + Ok(committee) => { + self.committees.insert(data.e3_id, committee); + } + Err(err) => { + error!( + e3_id = %data.e3_id, + "Finalized committee contains an invalid address: {err}; C0 keys will be rejected" + ); + } + } + } InterfoldEventData::EncryptionKeyReceived(data) => { self.notify_sync(ctx, TypedEvent::new(data, ec)) } + InterfoldEventData::E3RequestComplete(data) => { + let e3_id = data.e3_id; + self.presets.remove(&e3_id); + self.committees.remove(&e3_id); + self.pending + .retain(|(pending_e3, _), _| pending_e3 != &e3_id); + } _ => (), } } diff --git a/crates/zk-prover/src/domain/proof_verification.rs b/crates/zk-prover/src/domain/proof_verification.rs index 530d74f668..615f202dd2 100644 --- a/crates/zk-prover/src/domain/proof_verification.rs +++ b/crates/zk-prover/src/domain/proof_verification.rs @@ -11,7 +11,7 @@ //! consistency checks as a pure function. No actix / `BusHandle` concerns. use alloy::primitives::Address; -use e3_events::{Proof, SignedProofPayload}; +use e3_events::{E3id, Proof, ProofType, SignedProofPayload}; /// A validated external key, ready to be queued for ZK verification. #[derive(Debug)] @@ -24,8 +24,11 @@ pub(crate) struct ValidatedExternalKey { /// verification. /// /// Returns the cloned signed payload plus the recovered ECDSA signer address on -/// success, or a human-readable rejection reason. Signed proofs are mandatory. +/// success, or a human-readable rejection reason. Signed proofs are mandatory and are +/// bound to the outer E3, canonical committee party, C0 proof type, and advertised key bytes. pub(crate) fn validate_external_key( + expected_e3_id: &E3id, + expected_signer: &Address, party_id: u64, key_proof: Option<&Proof>, signed_payload: Option<&SignedProofPayload>, @@ -42,9 +45,29 @@ pub(crate) fn validate_external_key( )); }; + if signed.payload.e3_id != *expected_e3_id { + return Err(format!( + "Key from party {party_id} carries a C0 payload for E3 {}, expected {} - rejecting", + signed.payload.e3_id, expected_e3_id + )); + } + + if signed.payload.proof_type != ProofType::C0PkBfv { + return Err(format!( + "Key from party {party_id} carries proof type {:?}, expected {:?} - rejecting", + signed.payload.proof_type, + ProofType::C0PkBfv + )); + } + let recovered_signer = signed.recover_address().map_err(|err| { format!("Invalid signature on key from party {party_id} - rejecting: {err}") })?; + if recovered_signer != *expected_signer { + return Err(format!( + "Key from party {party_id} was signed by {recovered_signer}, expected committee member {expected_signer} - rejecting" + )); + } // Validate circuit name matches expected ProofType circuits. let expected_circuits = signed.payload.proof_type.circuit_names(); @@ -68,6 +91,26 @@ pub(crate) fn validate_external_key( }) } +/// Bind an already identity-validated C0 proof to the BFV key bytes advertised alongside it. +/// Kept separate so callers can reject unauthenticated payloads before computing the key +/// commitment, which is materially more expensive than signature and metadata checks. +pub(crate) fn validate_external_key_commitment( + party_id: u64, + proof: &Proof, + expected_key_commitment: &[u8; 32], +) -> Result<(), String> { + let proof_commitment = proof + .extract_output("pk_commitment") + .ok_or_else(|| format!("C0 proof from party {party_id} has no pk_commitment output"))?; + if &proof_commitment[..] != expected_key_commitment.as_slice() { + return Err(format!( + "C0 proof commitment does not match advertised BFV key for party {party_id} - rejecting" + )); + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -75,62 +118,146 @@ mod tests { use e3_events::{CircuitName, E3id, ProofPayload, ProofType}; use e3_utils::utility_types::ArcBytes; + const KEY_COMMITMENT: [u8; 32] = [0x42; 32]; + fn signer() -> PrivateKeySigner { "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" .parse() .unwrap() } + fn other_signer() -> PrivateKeySigner { + "0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d" + .parse() + .unwrap() + } + + fn e3_id() -> E3id { + E3id::new("1", 42) + } + fn proof() -> Proof { Proof::new( CircuitName::PkBfv, ArcBytes::from_bytes(&[10, 20, 30]), - ArcBytes::from_bytes(&[100, 200]), + ArcBytes::from_bytes(&KEY_COMMITMENT), ) } - fn signed_for(proof: &Proof) -> SignedProofPayload { + fn signed_for( + proof: &Proof, + e3_id: E3id, + proof_type: ProofType, + signer: &PrivateKeySigner, + ) -> SignedProofPayload { let payload = ProofPayload { - e3_id: E3id::new("1", 42), - proof_type: ProofType::C0PkBfv, + e3_id, + proof_type, proof: proof.clone(), }; - SignedProofPayload::sign(payload, &signer()).expect("signing should succeed") + SignedProofPayload::sign(payload, signer).expect("signing should succeed") + } + + fn valid_signed(proof: &Proof) -> SignedProofPayload { + signed_for(proof, e3_id(), ProofType::C0PkBfv, &signer()) + } + + fn validate( + proof: Option<&Proof>, + signed: Option<&SignedProofPayload>, + ) -> Result { + let validated = validate_external_key(&e3_id(), &signer().address(), 1, proof, signed)?; + validate_external_key_commitment( + 1, + proof.expect("validated proof exists"), + &KEY_COMMITMENT, + )?; + Ok(validated) } #[test] fn rejects_missing_proof() { let p = proof(); - let signed = signed_for(&p); - let err = validate_external_key(1, None, Some(&signed)).unwrap_err(); + let signed = valid_signed(&p); + let err = validate(None, Some(&signed)).unwrap_err(); assert!(err.contains("missing C0 proof")); } #[test] fn rejects_missing_signed_payload() { let p = proof(); - let err = validate_external_key(1, Some(&p), None).unwrap_err(); + let err = validate(Some(&p), None).unwrap_err(); assert!(err.contains("no signed payload")); } + #[test] + fn rejects_payload_for_different_e3() { + let p = proof(); + let signed = signed_for(&p, E3id::new("2", 42), ProofType::C0PkBfv, &signer()); + let err = validate(Some(&p), Some(&signed)).unwrap_err(); + assert!(err.contains("expected 42:1")); + } + + #[test] + fn rejects_non_c0_payload() { + let p = Proof::new( + CircuitName::PkGeneration, + ArcBytes::from_bytes(&[10, 20, 30]), + ArcBytes::from_bytes(&KEY_COMMITMENT), + ); + let signed = signed_for(&p, e3_id(), ProofType::C1PkGeneration, &signer()); + let err = validate(Some(&p), Some(&signed)).unwrap_err(); + assert!(err.contains("expected C0PkBfv")); + } + + #[test] + fn rejects_signer_that_does_not_own_party_slot() { + let p = proof(); + let signed = signed_for(&p, e3_id(), ProofType::C0PkBfv, &other_signer()); + let err = validate(Some(&p), Some(&signed)).unwrap_err(); + assert!(err.contains("expected committee member")); + } + #[test] fn rejects_proof_mismatch() { let p = proof(); - let signed = signed_for(&p); + let signed = valid_signed(&p); let other = Proof::new( CircuitName::PkBfv, ArcBytes::from_bytes(&[9, 9, 9]), - ArcBytes::from_bytes(&[1, 2]), + ArcBytes::from_bytes(&KEY_COMMITMENT), ); - let err = validate_external_key(1, Some(&other), Some(&signed)).unwrap_err(); + let err = validate(Some(&other), Some(&signed)).unwrap_err(); assert!(err.contains("Proof mismatch")); } + #[test] + fn rejects_proof_without_pk_commitment_output() { + let p = Proof::new( + CircuitName::PkBfv, + ArcBytes::from_bytes(&[10, 20, 30]), + ArcBytes::default(), + ); + let signed = valid_signed(&p); + let err = validate(Some(&p), Some(&signed)).unwrap_err(); + assert!(err.contains("no pk_commitment output")); + } + + #[test] + fn rejects_proof_for_different_key_bytes() { + let p = proof(); + let signed = valid_signed(&p); + let different_commitment = [0x24; 32]; + validate_external_key(&e3_id(), &signer().address(), 1, Some(&p), Some(&signed)).unwrap(); + let err = validate_external_key_commitment(1, &p, &different_commitment).unwrap_err(); + assert!(err.contains("does not match advertised BFV key")); + } + #[test] fn accepts_valid_key() { let p = proof(); - let signed = signed_for(&p); - let validated = validate_external_key(1, Some(&p), Some(&signed)).expect("should validate"); + let signed = valid_signed(&p); + let validated = validate(Some(&p), Some(&signed)).expect("should validate"); assert_eq!(validated.recovered_signer, signer().address()); } } From 35019a49fbbbc589c27cb99b76c4cab3ef665632 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:02:29 +0500 Subject: [PATCH 02/62] fix(slashing): use honest quorum for accusation votes --- crates/slashing/src/accusation_manager_ext.rs | 71 ++++++++++++-- .../slashing/src/actors/accusation_manager.rs | 98 +++++++++++++++++-- .../slashing/src/domain/accusation_voting.rs | 47 +++++---- 3 files changed, 178 insertions(+), 38 deletions(-) diff --git a/crates/slashing/src/accusation_manager_ext.rs b/crates/slashing/src/accusation_manager_ext.rs index 00417a3201..ca8f766590 100644 --- a/crates/slashing/src/accusation_manager_ext.rs +++ b/crates/slashing/src/accusation_manager_ext.rs @@ -7,8 +7,9 @@ //! E3Extension that wires up the [`AccusationManager`] per-E3 when the //! committee is finalized. //! -//! Listens for [`CommitteeFinalized`], reads `threshold_m` from [`E3Meta`], -//! parses committee addresses, and starts the actor with full context. +//! Listens for [`CommitteeFinalized`], derives the on-chain accusation quorum +//! from the circuit threshold in [`E3Meta`], parses committee addresses, and +//! starts the actor with full context. use std::collections::HashMap; @@ -19,8 +20,24 @@ use anyhow::Result; use async_trait::async_trait; use e3_events::{BusHandle, CommitteeFinalized, Event, InterfoldEvent, InterfoldEventData}; use e3_request::{E3Context, E3ContextSnapshot, E3Extension, META_KEY}; +use e3_zk_helpers::CiphernodesCommitteeSize; use tracing::{error, info, warn}; +/// Convert the compiled polynomial threshold `T` and committee size `N` into +/// the honest-party count `H` used by `SlashingManager` as its vote quorum. +/// +/// `E3Meta.threshold_m` intentionally carries `T` for circuit selection, while +/// the on-chain committee request and slashing contract use `H = T + 1`. Keep +/// both values separate so accusation voting agrees with Solidity without +/// breaking ZK re-verification artifact resolution. +fn accusation_vote_quorum(threshold_t: usize, committee_n: usize) -> Result { + Ok( + CiphernodesCommitteeSize::from_threshold(threshold_t, committee_n)? + .values() + .h, + ) +} + pub struct AccusationManagerExtension { bus: BusHandle, signer: PrivateKeySigner, @@ -103,29 +120,45 @@ impl E3Extension for AccusationManagerExtension { return; } - // Get threshold from meta + // `E3Meta` stores the compiled circuit threshold T. Solidity requires + // H votes, so derive and pass both values explicitly. let Some(meta) = ctx.get_dependency(META_KEY) else { error!("E3Meta not available — cannot start AccusationManager"); return; }; - let threshold_m = meta.threshold_m; + let circuit_threshold_t = meta.threshold_m; + let vote_quorum_h = match accusation_vote_quorum(meta.threshold_m, meta.threshold_n) { + Ok(quorum) => quorum, + Err(err) => { + error!( + %e3_id, + threshold_t = meta.threshold_m, + committee_n = meta.threshold_n, + error = %err, + "Unknown committee size — cannot start AccusationManager" + ); + return; + } + }; info!( - "Starting AccusationManager for E3 {} with {} committee members, threshold={}", + "Starting AccusationManager for E3 {} with {} committee members, circuit threshold T={}, vote quorum H={}", e3_id, committee_addresses.len(), - threshold_m + circuit_threshold_t, + vote_quorum_h ); let vote_validity_secs = self.vote_validity_secs_for(e3_id.chain_id()); - let addr = AccusationManager::setup( + let addr = AccusationManager::setup_with_quorum( &self.bus, e3_id, self.signer.clone(), self.slashing_manager, committee_addresses, - threshold_m, + circuit_threshold_t, + vote_quorum_h, vote_validity_secs, self.accusation_deadline_skew_secs, meta.params_preset, @@ -149,8 +182,28 @@ impl E3Extension for AccusationManagerExtension { /// - A malicious node cannot exploit restart-induced state loss to prevent slashing: /// restarting only loses *this node's* pending state — all other honest nodes still /// independently verify, vote, and reach quorum without this node's participation - /// (as long as enough honest nodes remain to meet threshold M). + /// (as long as enough honest nodes remain to meet the on-chain quorum H). async fn hydrate(&self, _ctx: &mut E3Context, _snapshot: &E3ContextSnapshot) -> Result<()> { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn accusation_quorum_matches_canonical_on_chain_committee_thresholds() { + for (threshold_t, committee_n, expected_h) in [(1, 3, 2), (4, 9, 5), (9, 19, 10)] { + assert_eq!( + accusation_vote_quorum(threshold_t, committee_n).unwrap(), + expected_h + ); + } + } + + #[test] + fn accusation_quorum_rejects_unknown_committee_parameters() { + assert!(accusation_vote_quorum(2, 3).is_err()); + } +} diff --git a/crates/slashing/src/actors/accusation_manager.rs b/crates/slashing/src/actors/accusation_manager.rs index f6c53a554e..7df032e82f 100644 --- a/crates/slashing/src/actors/accusation_manager.rs +++ b/crates/slashing/src/actors/accusation_manager.rs @@ -9,7 +9,7 @@ //! When a node detects a ZK proof failure from another committee member, it //! broadcasts a [`ProofFailureAccusation`] over gossip. Other committee members //! independently check the same proof and respond with [`AccusationVote`]s. -//! Once a quorum of M (the cryptographic threshold) votes is reached, the +//! Once the on-chain quorum `H` votes is reached, the //! actor emits [`AccusationQuorumReached`] for downstream consumers (aggregator //! exclusion, on-chain slash submission). //! @@ -49,7 +49,8 @@ use e3_events::{ ProofVerificationPassed, TypedEvent, }; use e3_utils::NotifySync; -use tracing::error; +use e3_zk_helpers::CiphernodesCommitteeSize; +use tracing::{error, warn}; use crate::domain::accusation_voting::{AccusationVoting, VoteAction}; @@ -115,6 +116,24 @@ pub struct AccusationManager { } impl AccusationManager { + fn canonical_vote_quorum(circuit_threshold_t: usize, committee_n: usize) -> usize { + match CiphernodesCommitteeSize::from_threshold(circuit_threshold_t, committee_n) { + Ok(size) => size.values().h, + Err(err) => { + // Preserve the historical constructor behavior for external callers with a + // non-canonical test committee. Production creation is validated by the + // extension and uses `setup_with_quorum` below. + warn!( + circuit_threshold_t, + committee_n, + error = %err, + "Unknown committee size; falling back to the supplied threshold as vote quorum" + ); + circuit_threshold_t + } + } + } + /// Construct an actor with the production [`SystemClock`]. Use /// [`AccusationManager::new_with_clock`] in tests that need deterministic /// timestamps. @@ -125,7 +144,7 @@ impl AccusationManager { signer: PrivateKeySigner, slashing_manager: Address, committee: Vec
, - threshold_m: usize, + circuit_threshold_t: usize, vote_validity_secs: u64, accusation_deadline_skew_secs: u64, params_preset: e3_fhe_params::BfvPreset, @@ -136,7 +155,7 @@ impl AccusationManager { signer, slashing_manager, committee, - threshold_m, + circuit_threshold_t, vote_validity_secs, accusation_deadline_skew_secs, params_preset, @@ -153,7 +172,37 @@ impl AccusationManager { signer: PrivateKeySigner, slashing_manager: Address, committee: Vec
, - threshold_m: usize, + circuit_threshold_t: usize, + vote_validity_secs: u64, + accusation_deadline_skew_secs: u64, + params_preset: e3_fhe_params::BfvPreset, + clock: Arc, + ) -> Self { + let vote_quorum_h = Self::canonical_vote_quorum(circuit_threshold_t, committee.len()); + Self::new_with_clock_and_quorum( + bus, + e3_id, + signer, + slashing_manager, + committee, + circuit_threshold_t, + vote_quorum_h, + vote_validity_secs, + accusation_deadline_skew_secs, + params_preset, + clock, + ) + } + + #[allow(clippy::too_many_arguments)] + fn new_with_clock_and_quorum( + bus: &BusHandle, + e3_id: E3id, + signer: PrivateKeySigner, + slashing_manager: Address, + committee: Vec
, + circuit_threshold_t: usize, + vote_quorum_h: usize, vote_validity_secs: u64, accusation_deadline_skew_secs: u64, params_preset: e3_fhe_params::BfvPreset, @@ -166,7 +215,8 @@ impl AccusationManager { signer, slashing_manager, committee, - threshold_m, + circuit_threshold_t, + vote_quorum_h, vote_validity_secs, accusation_deadline_skew_secs, params_preset, @@ -183,21 +233,51 @@ impl AccusationManager { signer: PrivateKeySigner, slashing_manager: Address, committee: Vec
, - threshold_m: usize, + circuit_threshold_t: usize, + vote_validity_secs: u64, + accusation_deadline_skew_secs: u64, + params_preset: e3_fhe_params::BfvPreset, + ) -> Addr { + let vote_quorum_h = Self::canonical_vote_quorum(circuit_threshold_t, committee.len()); + Self::setup_with_quorum( + bus, + e3_id, + signer, + slashing_manager, + committee, + circuit_threshold_t, + vote_quorum_h, + vote_validity_secs, + accusation_deadline_skew_secs, + params_preset, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn setup_with_quorum( + bus: &BusHandle, + e3_id: E3id, + signer: PrivateKeySigner, + slashing_manager: Address, + committee: Vec
, + circuit_threshold_t: usize, + vote_quorum_h: usize, vote_validity_secs: u64, accusation_deadline_skew_secs: u64, params_preset: e3_fhe_params::BfvPreset, ) -> Addr { - let addr = Self::new( + let addr = Self::new_with_clock_and_quorum( bus, e3_id, signer, slashing_manager, committee, - threshold_m, + circuit_threshold_t, + vote_quorum_h, vote_validity_secs, accusation_deadline_skew_secs, params_preset, + Arc::new(SystemClock), ) .start(); bus.subscribe(EventType::ProofVerificationFailed, addr.clone().into()); diff --git a/crates/slashing/src/domain/accusation_voting.rs b/crates/slashing/src/domain/accusation_voting.rs index 9434a09662..88edec1156 100644 --- a/crates/slashing/src/domain/accusation_voting.rs +++ b/crates/slashing/src/domain/accusation_voting.rs @@ -134,8 +134,10 @@ pub(crate) struct AccusationVoting { /// All committee member addresses for this E3. committee: Vec
, - /// Quorum threshold — matches the cryptographic threshold M. - threshold_m: usize, + /// Polynomial threshold `T`, used only for ZK circuit artifact resolution. + circuit_threshold_t: usize, + /// Vote quorum `H`, matching the on-chain `SlashingManager` threshold. + vote_quorum_h: usize, /// Original committee N fixed at construction for ZK circuit resolution. /// Do not derive from [`Self::committee`] after [`Self::on_slash_executed`] shrinks the roster. committee_n: usize, @@ -177,7 +179,8 @@ impl AccusationVoting { signer: PrivateKeySigner, slashing_manager: Address, committee: Vec
, - threshold_m: usize, + circuit_threshold_t: usize, + vote_quorum_h: usize, vote_validity_secs: u64, accusation_deadline_skew_secs: u64, params_preset: e3_fhe_params::BfvPreset, @@ -191,7 +194,8 @@ impl AccusationVoting { signer, slashing_manager, committee, - threshold_m, + circuit_threshold_t, + vote_quorum_h, committee_n, pending: HashMap::new(), accused_proofs: HashSet::new(), @@ -690,7 +694,7 @@ impl AccusationVoting { } } - // Check quorum immediately (in case threshold_m == 1) + // Check quorum immediately (defensive for a future H=1 committee). self.check_quorum(accusation_id, ec, actions); } @@ -848,7 +852,7 @@ impl AccusationVoting { let forwarded_clone = forwarded.clone(); let committee_size = match CiphernodesCommitteeSize::from_threshold( - self.threshold_m, + self.circuit_threshold_t, self.committee_n, ) { Ok(c) => c, @@ -1080,12 +1084,12 @@ impl AccusationVoting { }; let agree_count = pending.votes_for.len(); - if agree_count < self.threshold_m { + if agree_count < self.vote_quorum_h { // Not yet at quorum. return; } - // Reached `M` — decide between AccusedFaulted and Equivocation. + // Reached `H` — decide between AccusedFaulted and Equivocation. let agree_hashes: HashSet<[u8; 32]> = pending.votes_for.iter().map(|v| v.data_hash).collect(); if agree_hashes.len() > 1 { @@ -1120,7 +1124,7 @@ impl AccusationVoting { ) -> Option<(AccusationQuorumReached, EventContext)> { let pending = self.pending.remove(&accusation_id)?; // Already resolved - let outcome = if pending.votes_for.len() >= self.threshold_m { + let outcome = if pending.votes_for.len() >= self.vote_quorum_h { let agree_hashes: HashSet<[u8; 32]> = pending.votes_for.iter().map(|v| v.data_hash).collect(); if agree_hashes.len() > 1 { @@ -1380,7 +1384,8 @@ mod tests { fn voting_with( me: &PrivateKeySigner, committee: Vec
, - threshold_m: usize, + circuit_threshold_t: usize, + vote_quorum_h: usize, ) -> AccusationVoting { AccusationVoting::new( E3id::new("42", CHAIN_ID), @@ -1389,7 +1394,8 @@ mod tests { .parse() .unwrap(), committee, - threshold_m, + circuit_threshold_t, + vote_quorum_h, VALIDITY, SKEW, e3_fhe_params::BfvPreset::default(), @@ -1482,7 +1488,7 @@ mod tests { ); } - /// A second agreeing vote that reaches `threshold_m` must produce a single + /// A second agreeing vote that reaches `vote_quorum_h` must produce a single /// AccusedFaulted quorum decision and remove the pending accusation. #[test] fn tally_reaches_quorum_at_threshold() { @@ -1490,7 +1496,7 @@ mod tests { let b = signer(2); let accused = signer(9).address(); let committee = vec![me.address(), b.address(), accused]; - let mut v = voting_with(&me, committee, 2); + let mut v = voting_with(&me, committee, 1, 2); let sm = v.slashing_manager; let data_hash = [0x11; 32]; @@ -1523,7 +1529,7 @@ mod tests { let b = signer(2); let accused = signer(9).address(); let committee = vec![me.address(), b.address(), accused]; - let mut v = voting_with(&me, committee, 3); // threshold above what 2 votes reach + let mut v = voting_with(&me, committee, 1, 3); // quorum above what 2 votes reach let sm = v.slashing_manager; let data_hash = [0x11; 32]; @@ -1556,7 +1562,7 @@ mod tests { let c = signer(3); let accused = signer(9).address(); let committee = vec![me.address(), b.address(), c.address(), accused]; - let mut v = voting_with(&me, committee, 3); + let mut v = voting_with(&me, committee, 1, 3); let sm = v.slashing_manager; let data_hash = [0x11; 32]; @@ -1591,7 +1597,7 @@ mod tests { let b = signer(2); let accused = signer(9).address(); let committee = vec![me.address(), b.address(), accused]; - let mut v = voting_with(&me, committee, 2); + let mut v = voting_with(&me, committee, 1, 2); let sm = v.slashing_manager; let data_hash_a = [0x11; 32]; let data_hash_b = [0x22; 32]; @@ -1621,7 +1627,7 @@ mod tests { let me = signer(1); let accused = signer(9).address(); let committee = vec![me.address(), signer(2).address(), accused]; - let mut v = voting_with(&me, committee, 2); + let mut v = voting_with(&me, committee, 1, 2); let sm = v.slashing_manager; let data_hash = [0x11; 32]; @@ -1641,7 +1647,7 @@ mod tests { let me = signer(1); // Micro committee (T=4, N=9) — canonical pair in `from_threshold`. let committee: Vec
= (1..=9u8).map(|b| signer(b).address()).collect(); - let mut v = voting_with(&me, committee.clone(), 4); + let mut v = voting_with(&me, committee.clone(), 4, 5); let slashed = committee[8]; v.on_slash_executed(SlashExecuted { @@ -1656,10 +1662,11 @@ mod tests { assert_eq!(v.committee_n, 9); // Shrunken roster must not drive circuit resolution. assert!( - CiphernodesCommitteeSize::from_threshold(v.threshold_m, v.committee.len()).is_err() + CiphernodesCommitteeSize::from_threshold(v.circuit_threshold_t, v.committee.len()) + .is_err() ); assert_eq!( - CiphernodesCommitteeSize::from_threshold(v.threshold_m, v.committee_n).unwrap(), + CiphernodesCommitteeSize::from_threshold(v.circuit_threshold_t, v.committee_n).unwrap(), CiphernodesCommitteeSize::Micro ); } From 099ef9082303b3de6a51b7621bde76c5b8a083ea Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:02:40 +0500 Subject: [PATCH 03/62] fix(storage): fail closed on corrupt event log records --- Cargo.lock | 1 + crates/cli/src/node.rs | 8 +- crates/data/src/commit_log_event_log.rs | 164 ++++++++++----- crates/data/src/in_mem_event_log.rs | 32 +++ crates/entrypoint/Cargo.toml | 1 + crates/entrypoint/src/validate.rs | 258 ++++++++++++++++++------ crates/events/src/eventstore.rs | 243 ++++++++++++++++++---- crates/events/src/traits.rs | 12 ++ 8 files changed, 557 insertions(+), 162 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95ecb45b82..eae191c4a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3328,6 +3328,7 @@ dependencies = [ "anyhow", "bincode 1.3.3", "clap", + "commitlog", "dirs 5.0.1", "e3-aggregator", "e3-ciphernode-builder", diff --git a/crates/cli/src/node.rs b/crates/cli/src/node.rs index 75aecfdd41..22f5929c32 100644 --- a/crates/cli/src/node.rs +++ b/crates/cli/src/node.rs @@ -14,10 +14,10 @@ use e3_entrypoint::validate::validate_node; pub enum NodeCommands { /// Validate the on-disk state of a single node without starting it. /// - /// Opens the node's persisted stores read-only and checks that the schema - /// is loadable by this binary, the event log is intact, the snapshot cursor - /// is consistent, and there are no orphaned committee tickets ("loose - /// ends"). Safe to run while the node is stopped; intended as the + /// Takes the node's exclusive process fence and checks that the schema is + /// loadable by this binary, the event log is intact, the snapshot cursor is + /// consistent, and there are no orphaned committee tickets ("loose ends"). + /// Safe to run while the node is stopped; intended as the /// pre-upgrade and post-crash health check. Exits non-zero on failure. Validate, } diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 017625c535..c1aa726622 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -4,13 +4,11 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::Context; -use anyhow::Result; +use anyhow::{anyhow, Context, Result}; use commitlog::message::MessageSet; use commitlog::{CommitLog, LogOptions, ReadLimit}; use e3_events::{EventLog, InterfoldEvent, Unsequenced}; use std::path::PathBuf; -use tracing::error; /// Maximum message size for both reads and writes (32 MB). const MAX_MESSAGE_BYTES: usize = 32 * 1024 * 1024; @@ -36,71 +34,104 @@ impl CommitLogEventLog { // Return 1-indexed sequence number Ok(offset + 1) } -} -impl EventLog for CommitLogEventLog { - fn append(&mut self, event: &InterfoldEvent) -> Result { - let bytes = bincode::serialize(event)?; - self.append_bytes(&bytes) + /// Read and decode every event starting at `from`, failing on the first + /// unreadable commit-log segment or invalid event payload. + /// + /// A commit-log record with a valid frame/CRC but an invalid bincode payload + /// is not safe to treat as an ignorable torn tail. Its sequence number has + /// already been committed, so skipping it would make replayed state diverge + /// from the state that existed before the crash and a later append would turn + /// the same record into mid-log corruption. Callers that need a recoverable + /// integrity result (for example `interfold node validate`) use this method; + /// the [`EventLog`] adapter below fail-stops because its legacy iterator API + /// cannot carry an error. + pub fn read_from_checked(&self, from: u64) -> Result)>> { + self.read_from_checked_with_limit(from, None) } - #[allow(clippy::while_let_loop)] - fn read_from(&self, from: u64) -> Box)>> { - // Convert 1-indexed sequence to 0-indexed offset + fn read_from_checked_with_limit( + &self, + from: u64, + limit: Option, + ) -> Result)>> { + // Convert 1-indexed sequence to 0-indexed offset. let mut current_offset = from.saturating_sub(1); - let mut events = Vec::new(); - // Sequence number of the first message that failed to deserialize, if any. - // A deserialize failure is only tolerable when it is the *tail* of the log - // (a torn write from a crash mid-append). If a corrupt entry is followed by - // a valid one it is mid-log corruption and replaying past it would silently - // diverge actor state, so we halt loudly instead of skipping. - let mut corrupt_at: Option = None; + let mut events = Vec::with_capacity(limit.unwrap_or_default().min(1024)); + + if limit == Some(0) { + return Ok(events); + } loop { - let message_buf = match self + let message_buf = self .log .read(current_offset, ReadLimit::max_bytes(MAX_MESSAGE_BYTES)) - { - Ok(msgs) => msgs, - Err(_) => break, - }; + .map_err(|error| { + anyhow!( + "commit log read failed at sequence {}: {error:?}", + current_offset + 1 + ) + })?; let mut count = 0; for msg in message_buf.iter() { let seq = msg.offset() + 1; - match bincode::deserialize::>(msg.payload()) { - Ok(event) => { - if let Some(bad_seq) = corrupt_at { - // We already saw a corrupt entry and now found a valid one - // after it: the corruption is NOT at the tail. - panic!( - "Non-tail corruption in event log: entry at seq {bad_seq} failed \ - to deserialize but is followed by a valid entry at seq {seq}. \ - Replaying past it would silently drop an event. Halting; operator \ - recovery required." - ); - } - // Convert 0-indexed offset back to 1-indexed sequence number - events.push((seq, event)); - } - Err(_) => { - // Defer the decision: tolerate only if nothing valid follows. - error!("Error deserializing event in read_from at seq {seq}"); - if corrupt_at.is_none() { - corrupt_at = Some(seq); - } - } - } - current_offset = msg.offset() + 1; // Next offset to read from + let event = bincode::deserialize::>(msg.payload()) + .with_context(|| { + format!( + "commit log event at sequence {seq} failed to decode; log is corrupt" + ) + })?; + events.push((seq, event)); + current_offset = msg.offset() + 1; count += 1; + + if limit.is_some_and(|limit| events.len() >= limit) { + break; + } } - // No more messages to read - if count == 0 { + if count == 0 || limit.is_some_and(|limit| events.len() >= limit) { break; } } + Ok(events) + } +} + +impl EventLog for CommitLogEventLog { + fn append(&mut self, event: &InterfoldEvent) -> Result { + let bytes = bincode::serialize(event)?; + self.append_bytes(&bytes) + } + + fn read_from(&self, from: u64) -> Box)>> { + let events = self.read_from_checked(from).unwrap_or_else(|error| { + panic!( + "Event log integrity failure: {error:#}. Halting to prevent replay divergence; \ + run `interfold node validate` against the stopped node and restore from a \ + verified backup or resync." + ) + }); + Box::new(events.into_iter()) + } + + fn read_from_bounded( + &self, + from: u64, + limit: usize, + ) -> Box)>> { + let events = self + .read_from_checked_with_limit(from, Some(limit)) + .unwrap_or_else(|error| { + panic!( + "Event log integrity failure: {error:#}. Halting to prevent replay divergence; \ + run `interfold node validate` against the stopped node and restore from a \ + verified backup or resync." + ) + }); Box::new(events.into_iter()) } @@ -450,7 +481,22 @@ mod tests { } #[test] - fn test_read_from_corruption_at_end_causes_infinite_loop() { + fn bounded_read_decodes_only_requested_window() { + let dir = tempdir().unwrap(); + let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + for value in 1..=5 { + log.append(&event_from(TestEvent::new("event", value))) + .unwrap(); + } + + let events: Vec<_> = log.read_from_bounded(2, 2).collect(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].0, 2); + assert_eq!(events[1].0, 3); + } + + #[test] + fn read_from_checked_reports_tail_corruption() { let dir = tempdir().unwrap(); let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); @@ -461,12 +507,26 @@ mod tests { // Corrupt the last message log.append_bytes(b"I am a bad event!").unwrap(); - // Ensure if last message is corrupt we don't end up in an infinite loop + let error = log.read_from_checked(1).unwrap_err(); + let message = format!("{error:#}"); + assert!(message.contains("sequence 101"), "{message}"); + assert!(message.contains("failed to decode"), "{message}"); + } + + #[test] + #[should_panic(expected = "Event log integrity failure")] + fn event_log_adapter_fail_stops_on_tail_corruption() { + let dir = tempdir().unwrap(); + let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + + log.append(&event_from(TestEvent::new("valid", 1))).unwrap(); + log.append_bytes(b"I am a bad event!").unwrap(); + let _: Vec<_> = log.read_from(1).collect(); } #[test] - #[should_panic(expected = "Non-tail corruption")] + #[should_panic(expected = "Event log integrity failure")] fn test_read_from_non_tail_corruption_halts() { let dir = tempdir().unwrap(); let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index 67b735740b..571b8e76c3 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -37,6 +37,24 @@ impl EventLog for InMemEventLog { .collect(); Box::new(events.into_iter()) } + fn read_from_bounded( + &self, + from: u64, + limit: usize, + ) -> Box)>> { + // Convert 1-indexed sequence to 0-indexed array position and clone only the requested + // window. This path is used by bounded remote historical-sync queries. + let start_idx = from.saturating_sub(1) as usize; + let events: Vec<_> = self + .log + .iter() + .skip(start_idx) + .take(limit) + .enumerate() + .map(|(i, event)| (from + i as u64, event.clone())) + .collect(); + Box::new(events.into_iter()) + } fn append(&mut self, event: &InterfoldEvent) -> Result { self.log.push(event.to_owned()); Ok(self.log.len() as u64) @@ -100,6 +118,20 @@ mod tests { assert_eq!(events[1].0, 3); } + #[test] + fn bounded_read_clones_only_requested_window() { + let mut log = InMemEventLog::new(); + for value in 1..=5 { + log.append(&event_from(TestEvent::new("event", value))) + .unwrap(); + } + + let events: Vec<_> = log.read_from_bounded(2, 2).collect(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].0, 2); + assert_eq!(events[1].0, 3); + } + #[test] fn test_read_empty_log() { let log = InMemEventLog::new(); diff --git a/crates/entrypoint/Cargo.toml b/crates/entrypoint/Cargo.toml index e37c603e82..9f7ff0911c 100644 --- a/crates/entrypoint/Cargo.toml +++ b/crates/entrypoint/Cargo.toml @@ -50,4 +50,5 @@ reqwest = { workspace = true } serde_json = { workspace = true } [dev-dependencies] +commitlog = { workspace = true } tempfile = { workspace = true } diff --git a/crates/entrypoint/src/validate.rs b/crates/entrypoint/src/validate.rs index 683c77baf5..69b176bc11 100644 --- a/crates/entrypoint/src/validate.rs +++ b/crates/entrypoint/src/validate.rs @@ -7,12 +7,12 @@ //! Offline node-state validation. //! //! Backs the `interfold node validate` CLI command. It opens a node's persisted -//! stores **read-only** (no actors, no network, no chain writes) and answers the +//! stores offline (no network or chain writes) and answers the //! operator question: *"Is my on-disk state intact, internally consistent, free //! of loose ends, and will this binary be able to load it after an upgrade?"* //! -//! It is deliberately non-destructive. It never mutates the store, never talks to -//! the chain, and never starts the node. It is safe to run while the node is +//! It is deliberately non-destructive. It never mutates protocol state, never +//! talks to the chain, and never starts the node. It is safe to run while the node is //! stopped (the recommended pre-upgrade step) and surfaces problems as a //! structured report with a non-zero exit on failure. //! @@ -31,19 +31,21 @@ //! orphaned tickets that a crash mid-E3 can leave behind; they are the //! "loose ends" a restart should clean up. -use crate::helpers::datastore::{get_eventstore_reader, get_repositories}; -use anyhow::{anyhow, Result}; -use e3_ciphernode_builder::global_eventstore_cache::EventStoreReader; +use crate::helpers::datastore::get_repositories; +use anyhow::{bail, Context, Result}; use e3_config::AppConfig; -use e3_data::Repositories; +use e3_data::{CommitLogEventLog, Repositories}; use e3_events::{ - AggregateId, CorrelationId, E3Stage, Event, EventContextAccessors, EventContextSeq, - EventStoreQueryBy, EventStoreQueryResponse, InterfoldEvent, InterfoldEventData, SeqAgg, + AggregateId, E3Stage, Event, EventContextAccessors, EventContextSeq, InterfoldEvent, + InterfoldEventData, }; use e3_sortition::{committee_key, NodeRegistry, NodeStateRepositoryFactory, NodeStateStore}; -use e3_sync::SyncRepositoryFactory; -use e3_utils::actix::channel as actix_toolbox; +use e3_sync::{ + decide_schema_version, SchemaVersionDecision, SyncRepositoryFactory, SCHEMA_VERSION, +}; +use e3_utils::enumerate_path; use std::collections::{BTreeMap, HashMap, HashSet}; +use std::path::Path; /// Outcome severity for a single validation check. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -153,27 +155,67 @@ impl ValidationReport { /// Run every validation check against the node configured by `config`. /// -/// Opens the persisted stores read-only. Returns the full report; callers decide -/// how to surface it (the CLI prints it and exits non-zero on failure). +/// Opens the persisted stores while holding the node's exclusive process fence. +/// Returns the full report; callers decide how to surface it (the CLI prints it +/// and exits non-zero on failure). pub async fn validate_node(config: &AppConfig) -> Result { - let repositories = get_repositories(config)?; - let eventstore = get_eventstore_reader(config)?; let aggregate_ids = aggregate_ids(config); - let mut report = ValidationReport::default(); - // 1 + 2. Read every event per aggregate, check integrity + cursor consistency, - // and collect the terminal-event keys for the open-loop audit. + // 1. Read the commit logs directly before starting any EventStore actor. The + // actor-facing iterator intentionally fail-stops on corruption; the checked + // reader used here lets the operator receive a structured validation report. let mut terminal_keys: HashSet = HashSet::new(); let mut total_events: u64 = 0; + let mut events_by_aggregate = Vec::with_capacity(aggregate_ids.len()); + let mut unreadable_logs = 0usize; for agg in &aggregate_ids { - let events = read_all_events(&eventstore, *agg).await?; - total_events += events.len() as u64; + let path = enumerate_path(&config.log_file(), agg.to_usize()); + match read_event_log(&path, *agg) { + Ok(events) => { + total_events += events.len() as u64; + collect_terminal_keys(&events, &mut terminal_keys); + + let seqs: Vec = events.iter().map(|e| e.seq()).collect(); + report.push(check_sequence_integrity(*agg, &seqs)); + events_by_aggregate.push((*agg, events)); + } + Err(error) => { + unreadable_logs += 1; + report.push(CheckResult::fail( + "event-log", + format!( + "aggregate {} at {} is unreadable or corrupt: {error:#}", + agg.to_usize(), + path.display() + ), + )); + } + } + } - collect_terminal_keys(&events, &mut terminal_keys); + if unreadable_logs > 0 { + report.push(CheckResult::fail( + "event-store", + format!( + "{unreadable_logs} of {} aggregate log(s) could not be decoded; snapshot and \ + open-loop checks were skipped because their inputs are incomplete", + aggregate_ids.len() + ), + )); + return Ok(report); + } + // 2. Only open the snapshot store after every source-of-truth log passed its + // framing and decode checks. Cross-check each persisted replay cursor. + let repositories = get_repositories(config)?; + let persisted_schema = repositories.schema_version().read().await?; + report.push(check_schema_compatibility( + persisted_schema, + total_events > 0, + )); + for (agg, events) in &events_by_aggregate { let seqs: Vec = events.iter().map(|e| e.seq()).collect(); - report.push(check_sequence_integrity(*agg, &seqs)); let cursor = repositories.aggregate_seq(*agg).read().await?.unwrap_or(0); report.push(check_cursor_consistency(*agg, cursor, &seqs)); @@ -192,6 +234,32 @@ pub async fn validate_node(config: &AppConfig) -> Result { Ok(report) } +/// Verify that this binary can safely interpret the persisted schema. A missing +/// marker is acceptable only for a genuinely empty store; stamping a version on +/// non-empty legacy bytes would assert compatibility without evidence. +fn check_schema_compatibility(persisted: Option, has_events: bool) -> CheckResult { + let name = "schema"; + match decide_schema_version(persisted, SCHEMA_VERSION) { + SchemaVersionDecision::Proceed => CheckResult::pass( + name, + format!("on-disk schema version {SCHEMA_VERSION} matches this binary"), + ), + SchemaVersionDecision::WriteCurrent if !has_events => CheckResult::pass( + name, + format!("empty store will be initialized at schema version {SCHEMA_VERSION}"), + ), + SchemaVersionDecision::WriteCurrent => CheckResult::fail( + name, + format!( + "non-empty event log has no schema marker; compatibility with schema version \ + {SCHEMA_VERSION} cannot be proven. Back up the stopped node and use an explicit \ + migration or controlled resync" + ), + ), + SchemaVersionDecision::Halt(reason) => CheckResult::fail(name, reason), + } +} + /// The set of aggregate ids to inspect: the local aggregate (0) plus one per /// configured chain. Mirrors [`AggregateId::from_chain_id`] so the validator /// looks at exactly the aggregates the running node persists. @@ -423,49 +491,38 @@ fn collect_terminal_keys(events: &[InterfoldEvent], out: &mut HashSet) { } } -/// Read every event for a single aggregate from sequence 0, paginating until the -/// store is exhausted. -async fn read_all_events( - eventstore: &EventStoreReader, - aggregate: AggregateId, -) -> Result> { - const PAGE: u64 = 1024; - let mut all: Vec = Vec::new(); - let mut since: u64 = 0; - loop { - let (addr, rx) = actix_toolbox::oneshot::(); - let msg = EventStoreQueryBy::::new( - CorrelationId::new(), - HashMap::from([(aggregate, since)]), - addr, - ) - .with_limit(PAGE); - eventstore - .seq() - .try_send(msg) - .map_err(|e| anyhow!("event store query failed: {e}"))?; - let page = rx.await?.into_events(); - if page.is_empty() { - break; - } - let max_seq = page.iter().map(|e| e.seq()).max().unwrap_or(since); - all.extend(page); - // Advance past the highest sequence we just read. If the store could not - // advance (defensive), stop to avoid an infinite loop. - let next = max_seq.saturating_add(1); - if next <= since { - break; - } - since = next; - } - // The store may interleave aggregates depending on the query; keep only this - // aggregate's events and order them by sequence for the integrity check. Do not - // deduplicate by seq — repeated sequence numbers are exactly the corruption the - // integrity check must surface (pagination advances strictly past the last seq, - // so it never produces legitimate duplicates). - all.retain(|e| e.aggregate_id() == aggregate); - all.sort_by_key(|e| e.seq()); - Ok(all) +/// Read one aggregate's source-of-truth commit log without creating an empty log +/// as a side effect when the node has never persisted that aggregate. +fn read_event_log(path: &Path, expected_aggregate: AggregateId) -> Result> { + if !path.exists() { + return Ok(Vec::new()); + } + + let log = CommitLogEventLog::new(&path.to_path_buf()) + .with_context(|| format!("failed to open commit log {}", path.display()))?; + let events: Vec = log + .read_from_checked(1) + .with_context(|| format!("failed integrity scan for {}", path.display())) + .map(|events| { + events + .into_iter() + .map(|(seq, event)| event.into_sequenced(seq)) + .collect() + })?; + + if let Some(event) = events + .iter() + .find(|event| event.aggregate_id() != expected_aggregate) + { + bail!( + "event at sequence {} belongs to aggregate {}, but log path is for aggregate {}", + event.seq(), + event.aggregate_id().to_usize(), + expected_aggregate.to_usize() + ); + } + + Ok(events) } /// A non-empty `BTreeMap` alias kept for readability in tests. @@ -475,7 +532,78 @@ type SeqMap = BTreeMap; #[cfg(test)] mod tests { use super::*; + use commitlog::{CommitLog, LogOptions}; + use e3_events::{EventConstructorWithTimestamp, EventLog, EventSource, TestEvent, Unsequenced}; use e3_sortition::OpenCommittee; + use tempfile::tempdir; + + #[test] + fn validator_reader_reports_corrupt_tail_instead_of_skipping_it() { + let dir = tempdir().unwrap(); + let log_path = dir.path().join("log.0"); + let mut raw_log = CommitLog::new(LogOptions::new(&log_path)).unwrap(); + raw_log + .append_msg(b"valid commit-log frame, invalid event payload") + .unwrap(); + drop(raw_log); + + let error = read_event_log(&log_path, AggregateId::new(0)).unwrap_err(); + let message = format!("{error:#}"); + assert!(message.contains("sequence 1"), "{message}"); + assert!(message.contains("failed to decode"), "{message}"); + } + + #[test] + fn validator_reader_does_not_create_a_missing_log() { + let dir = tempdir().unwrap(); + let log_path = dir.path().join("log.0"); + + assert!(read_event_log(&log_path, AggregateId::new(0)) + .unwrap() + .is_empty()); + assert!(!log_path.exists()); + } + + #[test] + fn validator_reader_rejects_event_in_wrong_aggregate_log() { + let dir = tempdir().unwrap(); + let log_path = dir.path().join("log.1"); + let mut log = CommitLogEventLog::new(&log_path).unwrap(); + let aggregate_zero_event = InterfoldEvent::::new_with_timestamp( + TestEvent::new("misfiled", 1).into(), + None, + 1, + None, + EventSource::Local, + ); + log.append(&aggregate_zero_event).unwrap(); + drop(log); + + let error = read_event_log(&log_path, AggregateId::new(1)).unwrap_err(); + let message = error.to_string(); + assert!(message.contains("belongs to aggregate 0"), "{message}"); + assert!(message.contains("for aggregate 1"), "{message}"); + } + + #[test] + fn schema_check_accepts_exact_version() { + let result = check_schema_compatibility(Some(SCHEMA_VERSION), true); + assert_eq!(result.severity, Severity::Pass); + } + + #[test] + fn schema_check_rejects_missing_marker_on_nonempty_log() { + let result = check_schema_compatibility(None, true); + assert_eq!(result.severity, Severity::Fail); + assert!(result.detail.contains("no schema marker")); + } + + #[test] + fn schema_check_rejects_incompatible_version() { + let result = check_schema_compatibility(Some(SCHEMA_VERSION + 1), true); + assert_eq!(result.severity, Severity::Fail); + assert!(result.detail.contains("newer")); + } #[test] fn sequence_ok_when_contiguous() { diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 7ab4ea0821..9f6e9175b5 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -13,8 +13,8 @@ use actix::{Actor, Handler}; use anyhow::{bail, Result}; use tracing::{error, warn}; +const INDEX_RECONCILE_PAGE_SIZE: usize = 1_024; const MAX_STORAGE_ERRORS: u64 = 10; - pub struct EventStore { index: I, log: L, @@ -78,8 +78,17 @@ impl EventStore { let Some(seq) = self.index.seek(query)? else { return Ok(vec![]); }; - let events: Vec<_> = self.log.read_from(seq).collect(); - let result = self.collect_events(Box::new(events.into_iter()), filter, limit); + // For unfiltered queries, push the limit down to the log implementation. Historical net + // sync uses this path and must not read/materialize the complete remaining history for a + // single remote request. A source-filtered query cannot safely apply the limit before the + // filter without changing its semantics, so it retains the unbounded iterator path. + let events = match (filter.as_ref(), limit) { + (None, Some(limit)) => self + .log + .read_from_bounded(seq, usize::try_from(limit).unwrap_or(usize::MAX)), + _ => self.log.read_from(seq), + }; + let result = self.collect_events(events, filter, limit); Ok(result) } @@ -131,15 +140,62 @@ impl EventStore { return; } let mut repaired = 0u64; - for (seq, event) in self.log.read_from(1) { - let ts = event.ts(); - match self.index.get(ts) { - Ok(Some(_)) => {} - Ok(None) => match self.index.insert(ts, seq) { - Ok(()) => repaired += 1, - Err(e) => error!("Failed to backfill event index at seq {seq}: {e}"), - }, - Err(e) => error!("Failed to read event index at ts {ts}: {e}"), + let mut next_sequence = 1u64; + while next_sequence <= head { + let mut page_len = 0usize; + for (seq, event) in self + .log + .read_from_bounded(next_sequence, INDEX_RECONCILE_PAGE_SIZE) + { + if seq != next_sequence { + panic!( + "Event log integrity failure during index reconciliation: expected sequence \ + {next_sequence}, got {seq}. Halting; run `interfold node validate` and \ + recover from a verified backup or controlled resync." + ); + } + if seq > head { + panic!( + "Event log changed during index reconciliation: captured head {head}, but \ + bounded read returned sequence {seq}. Halting to avoid an inconsistent \ + derived index." + ); + } + + let ts = event.ts(); + match self.index.get(ts) { + Ok(Some(_)) => {} + Ok(None) => match self.index.insert(ts, seq) { + Ok(()) => repaired += 1, + Err(error) => panic!( + "Failed to backfill event index at sequence {seq}: {error}. Halting to \ + avoid serving an incomplete timestamp index." + ), + }, + Err(error) => panic!( + "Failed to read event index at timestamp {ts}: {error}. Halting to avoid \ + serving an unverified timestamp index." + ), + } + next_sequence = seq.checked_add(1).unwrap_or_else(|| { + panic!("Event log sequence overflow while reconciling sequence {seq}") + }); + page_len += 1; + } + + if page_len == 0 { + panic!( + "Event log integrity failure during index reconciliation: log head is {head}, \ + but no event was returned for sequence {next_sequence}. Halting; run \ + `interfold node validate` and recover from a verified backup or controlled \ + resync." + ); + } + if page_len > INDEX_RECONCILE_PAGE_SIZE { + panic!( + "Event log violated bounded-read contract during index reconciliation: \ + returned {page_len} events for limit {INDEX_RECONCILE_PAGE_SIZE}." + ); } } if repaired > 0 { @@ -221,7 +277,13 @@ mod tests { use super::*; use anyhow::Result; - use std::collections::BTreeMap; + use std::{ + collections::BTreeMap, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + }; // --------------------------------------------------------------------------- // Mock SequenceIndex backed by BTreeMap @@ -252,37 +314,94 @@ mod tests { // --------------------------------------------------------------------------- // Mock EventLog backed by Vec // --------------------------------------------------------------------------- - struct MockLog(Vec>); + struct MockLog { + events: Vec>, + bounded_read_calls: Option>, + bounded_read_limit: Option>, + unbounded_read_calls: Option>, + } impl MockLog { fn new() -> Self { - Self(Vec::new()) + Self { + events: Vec::new(), + bounded_read_calls: None, + bounded_read_limit: None, + unbounded_read_calls: None, + } + } + + fn with_bounded_read_tracker(tracker: Arc) -> Self { + Self { + events: Vec::new(), + bounded_read_calls: None, + bounded_read_limit: Some(tracker), + unbounded_read_calls: None, + } + } + + fn with_reconcile_trackers( + events: Vec>, + bounded_read_calls: Arc, + bounded_read_limit: Arc, + unbounded_read_calls: Arc, + ) -> Self { + Self { + events, + bounded_read_calls: Some(bounded_read_calls), + bounded_read_limit: Some(bounded_read_limit), + unbounded_read_calls: Some(unbounded_read_calls), + } } } impl EventLog for MockLog { fn append(&mut self, event: &InterfoldEvent) -> Result { - let seq = self.0.len() as u64; - self.0.push(event.clone()); - Ok(seq) + self.events.push(event.clone()); + Ok(self.events.len() as u64) } fn read_from( &self, from: u64, ) -> Box)>> { + if let Some(calls) = &self.unbounded_read_calls { + calls.fetch_add(1, Ordering::SeqCst); + } let items: Vec<_> = self - .0 + .events .iter() .enumerate() - .filter(move |(i, _)| *i as u64 >= from) - .map(|(i, e)| (i as u64, e.clone())) + .filter(move |(i, _)| (*i as u64 + 1) >= from) + .map(|(i, e)| (i as u64 + 1, e.clone())) + .collect(); + Box::new(items.into_iter()) + } + + fn read_from_bounded( + &self, + from: u64, + limit: usize, + ) -> Box)>> { + if let Some(calls) = &self.bounded_read_calls { + calls.fetch_add(1, Ordering::SeqCst); + } + if let Some(tracker) = &self.bounded_read_limit { + tracker.fetch_max(limit, Ordering::SeqCst); + } + let items: Vec<_> = self + .events + .iter() + .enumerate() + .filter(move |(i, _)| (*i as u64 + 1) >= from) + .take(limit) + .map(|(i, event)| (i as u64 + 1, event.clone())) .collect(); Box::new(items.into_iter()) } fn head(&self) -> u64 { - self.0.len().saturating_sub(1) as u64 + self.events.len() as u64 } } @@ -341,9 +460,9 @@ mod tests { let _b = store.store_event(make_local_event(200)).unwrap().unwrap(); let _c = store.store_event(make_local_event(300)).unwrap().unwrap(); - assert_eq!(store.index.get(100).unwrap(), Some(0)); - assert_eq!(store.index.get(200).unwrap(), Some(1)); - assert_eq!(store.index.get(300).unwrap(), Some(2)); + assert_eq!(store.index.get(100).unwrap(), Some(1)); + assert_eq!(store.index.get(200).unwrap(), Some(2)); + assert_eq!(store.index.get(300).unwrap(), Some(3)); } #[test] @@ -352,10 +471,37 @@ mod tests { store.store_event(make_local_event(100)).unwrap(); store.store_event(make_local_event(200)).unwrap(); - let logged: Vec<_> = store.log.read_from(0).collect(); + let logged: Vec<_> = store.log.read_from(1).collect(); assert_eq!(logged.len(), 2); } + #[test] + fn startup_index_reconciliation_reads_log_in_bounded_pages() { + let event_count = INDEX_RECONCILE_PAGE_SIZE * 2 + 2; + let events = (1..=event_count) + .map(|timestamp| make_local_event(timestamp as u128)) + .collect::>(); + let bounded_calls = Arc::new(AtomicUsize::new(0)); + let max_limit = Arc::new(AtomicUsize::new(0)); + let unbounded_calls = Arc::new(AtomicUsize::new(0)); + let log = MockLog::with_reconcile_trackers( + events, + Arc::clone(&bounded_calls), + Arc::clone(&max_limit), + Arc::clone(&unbounded_calls), + ); + + let store = EventStore::new(MockIndex::new(), log); + + assert_eq!(unbounded_calls.load(Ordering::SeqCst), 0); + assert_eq!(bounded_calls.load(Ordering::SeqCst), 3); + assert_eq!(max_limit.load(Ordering::SeqCst), INDEX_RECONCILE_PAGE_SIZE); + assert_eq!( + store.index.get(event_count as u128).unwrap(), + Some(event_count as u64) + ); + } + #[test] fn store_event_returns_none_for_duplicate_timestamp() { let mut store = new_store(); @@ -401,7 +547,7 @@ mod tests { make_local_event(300), ]); - let events = store.query_by_seq(0, None, None); + let events = store.query_by_seq(1, None, None); assert_eq!(events.len(), 3); } @@ -415,7 +561,7 @@ mod tests { make_local_event(400), ]); - let events = store.query_by_seq(2, None, None); + let events = store.query_by_seq(3, None, None); assert_eq!(events.len(), 2); } @@ -430,7 +576,7 @@ mod tests { ]); let events = - store.query_by_seq(0, Some(EventStoreFilter::Source(EventSource::Local)), None); + store.query_by_seq(1, Some(EventStoreFilter::Source(EventSource::Local)), None); assert_eq!(events.len(), 2); for e in &events { @@ -448,7 +594,7 @@ mod tests { make_local_event(500), ]); - let events = store.query_by_seq(0, None, Some(2)); + let events = store.query_by_seq(1, None, Some(2)); assert_eq!(events.len(), 2); } @@ -464,7 +610,7 @@ mod tests { ]); let events = store.query_by_seq( - 0, + 1, Some(EventStoreFilter::Source(EventSource::Local)), Some(2), ); @@ -479,7 +625,7 @@ mod tests { fn seq_query_on_empty_log_returns_empty() { let store = new_store(); - let events = store.query_by_seq(0, None, None); + let events = store.query_by_seq(1, None, None); assert!(events.is_empty()); } @@ -564,6 +710,21 @@ mod tests { assert_eq!(events.len(), 2); } + #[test] + fn ts_query_pushes_unfiltered_limit_into_event_log() { + let observed_limit = Arc::new(AtomicUsize::new(0)); + let log = MockLog::with_bounded_read_tracker(observed_limit.clone()); + let mut store = EventStore::new(MockIndex::new(), log); + for timestamp in [100, 200, 300, 400] { + store.store_event(make_local_event(timestamp)).unwrap(); + } + + let events = store.query_by_ts(100, None, Some(2)).unwrap(); + + assert_eq!(events.len(), 2); + assert_eq!(observed_limit.load(Ordering::SeqCst), 2); + } + // =========================================================================== // Pagination / Cursor boundary tests // =========================================================================== @@ -651,15 +812,15 @@ mod tests { make_local_event(300), ]); - let page1 = store.query_by_seq(0, None, Some(2)); + let page1 = store.query_by_seq(1, None, Some(2)); assert_eq!(page1.len(), 2); - assert_eq!(page1[0].seq(), 0); - assert_eq!(page1[1].seq(), 1); + assert_eq!(page1[0].seq(), 1); + assert_eq!(page1[1].seq(), 2); - let page2 = store.query_by_seq(1, None, Some(2)); + let page2 = store.query_by_seq(2, None, Some(2)); assert_eq!(page2.len(), 2); - assert_eq!(page2[0].seq(), 1); - assert_eq!(page2[1].seq(), 2); + assert_eq!(page2[0].seq(), 2); + assert_eq!(page2[1].seq(), 3); } #[test] @@ -670,11 +831,11 @@ mod tests { make_local_event(300), ]); - let page1 = store.query_by_seq(0, None, Some(2)); + let page1 = store.query_by_seq(1, None, Some(2)); assert_eq!(page1.len(), 2); let cursor_seq = page1.last().unwrap().seq(); - assert_eq!(cursor_seq, 1); + assert_eq!(cursor_seq, 2); let page2 = store.query_by_seq(cursor_seq, None, None); assert_eq!(page2.len(), 2); @@ -689,7 +850,7 @@ mod tests { assert!( has_duplicates, - "BUG: seq=1 appears in both pages (inclusive query with cursor=last_seq)" + "BUG: seq=2 appears in both pages (inclusive query with cursor=last_seq)" ); } } diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 7047c994b3..fc5d7843e7 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -179,6 +179,18 @@ pub trait EventLog: Unpin + 'static { fn append(&mut self, event: &InterfoldEvent) -> Result; /// Read all events starting from the given sequence number (inclusive) fn read_from(&self, from: u64) -> Box)>>; + /// Read at most `limit` events starting from the given sequence number (inclusive). + /// + /// Implementations backed by persistent storage should override this method so a bounded + /// query does not materialize the complete remaining log before truncating it. The default + /// preserves compatibility for lightweight/custom implementations. + fn read_from_bounded( + &self, + from: u64, + limit: usize, + ) -> Box)>> { + Box::new(self.read_from(from).take(limit)) + } /// The 1-indexed sequence number of the last appended event, or `0` if the log is empty. fn head(&self) -> u64; } From 11a756fbd35ac7ec58455913c62424a70b23aba0 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:02:47 +0500 Subject: [PATCH 04/62] fix(net): keep broadcast consumers alive after lag --- crates/net/src/actors/document_publisher.rs | 4 +- crates/net/src/actors/mod.rs | 97 +++++++++++++++++++ crates/net/src/actors/net_event_translator.rs | 2 +- crates/net/src/actors/net_sync_manager.rs | 2 +- 4 files changed, 102 insertions(+), 3 deletions(-) diff --git a/crates/net/src/actors/document_publisher.rs b/crates/net/src/actors/document_publisher.rs index 00c4c28df5..0e47fdb690 100644 --- a/crates/net/src/actors/document_publisher.rs +++ b/crates/net/src/actors/document_publisher.rs @@ -102,7 +102,9 @@ impl DocumentPublisher { debug!("Spawning event receive loop!"); let addr = addr.clone(); async move { - while let Ok(event) = events.recv().await { + while let Some(event) = + super::recv_net_event(&mut events, "DocumentPublisher").await + { debug!("Received event {:?}", event); if let NetEvent::GossipData(GossipData::DocumentPublishedNotification(data)) = event diff --git a/crates/net/src/actors/mod.rs b/crates/net/src/actors/mod.rs index 5b556473a7..f3cd62633d 100644 --- a/crates/net/src/actors/mod.rs +++ b/crates/net/src/actors/mod.rs @@ -7,6 +7,10 @@ //! Thin actix actor shells. These contain no business logic — they wire bus/network events to the //! pure services in [`crate::domain`] and perform the resulting I/O. +use crate::events::NetEvent; +use tokio::sync::broadcast::{error::RecvError, Receiver}; +use tracing::{debug, warn}; + mod document_publisher; mod net_event_buffer; mod net_event_translator; @@ -21,3 +25,96 @@ pub use net_event_translator::NetEventTranslator; // Internal wiring helpers used by `setup_net`; not part of the public API. pub(crate) use net_event_buffer::NetEventBuffer; pub(crate) use net_sync_manager::NetSyncManager; + +/// Receive the next retained network event without treating broadcast lag as channel closure. +/// +/// Tokio reports lag as an item-level error and advances the receiver to the oldest event still in +/// the ring buffer. Callers must therefore retry after `Lagged`; returning from their receive task +/// would permanently detach that networking subsystem after a single burst. Only `Closed` is +/// terminal. The warning deliberately contains only a static consumer name and a numeric count so +/// peer- or payload-controlled data cannot create unbounded log fields. +async fn recv_net_event( + receiver: &mut Receiver, + consumer: &'static str, +) -> Option { + loop { + match receiver.recv().await { + Ok(event) => return Some(event), + Err(RecvError::Lagged(skipped_events)) => { + warn!( + consumer, + skipped_events, "network event receiver lagged; resuming from retained events" + ); + } + Err(RecvError::Closed) => { + debug!(consumer, "network event broadcast channel closed"); + return None; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tokio::sync::broadcast::error::TryRecvError; + + #[tokio::test] + async fn lagged_receiver_continues_with_retained_and_future_events() { + let (tx, mut receiver) = tokio::sync::broadcast::channel(1); + let mut lag_sentinel_control = tx.subscribe(); + + tx.send(NetEvent::AllPeersDialed { + connected: 1, + total: 2, + }) + .expect("receiver is active"); + tx.send(NetEvent::AllPeersDialed { + connected: 2, + total: 2, + }) + .expect("receiver is active"); + + assert!(matches!( + lag_sentinel_control.try_recv(), + Err(TryRecvError::Lagged(skipped)) if skipped > 0 + )); + + let retained = recv_net_event(&mut receiver, "test-consumer") + .await + .expect("lag must not terminate the receiver"); + assert!(matches!( + retained, + NetEvent::AllPeersDialed { + connected: 2, + total: 2 + } + )); + + tx.send(NetEvent::AllPeersDialed { + connected: 3, + total: 3, + }) + .expect("receiver remains active after lag"); + let future = recv_net_event(&mut receiver, "test-consumer") + .await + .expect("receiver must continue processing future events"); + assert!(matches!( + future, + NetEvent::AllPeersDialed { + connected: 3, + total: 3 + } + )); + } + + #[tokio::test] + async fn closed_receiver_is_terminal() { + let (tx, mut receiver) = tokio::sync::broadcast::channel::(1); + drop(tx); + + assert!(recv_net_event(&mut receiver, "test-consumer") + .await + .is_none()); + } +} diff --git a/crates/net/src/actors/net_event_translator.rs b/crates/net/src/actors/net_event_translator.rs index 4903828832..fd002e9ccc 100644 --- a/crates/net/src/actors/net_event_translator.rs +++ b/crates/net/src/actors/net_event_translator.rs @@ -63,7 +63,7 @@ impl NetEventTranslator { tokio::spawn({ let addr = addr.clone(); async move { - while let Ok(event) = rx.recv().await { + while let Some(event) = super::recv_net_event(&mut rx, "NetEventTranslator").await { if let NetEvent::GossipData(data) = event { if let GossipData::GossipBytes(_) = data { addr.do_send(LibP2pEvent(data)); diff --git a/crates/net/src/actors/net_sync_manager.rs b/crates/net/src/actors/net_sync_manager.rs index 0a81924e94..a7630f4b13 100644 --- a/crates/net/src/actors/net_sync_manager.rs +++ b/crates/net/src/actors/net_sync_manager.rs @@ -236,7 +236,7 @@ impl NetSyncManager { debug!("Spawning event receive loop!"); let addr = addr.clone(); async move { - while let Ok(event) = events.recv().await { + while let Some(event) = super::recv_net_event(&mut events, "NetSyncManager").await { debug!("Received event {:?}", event); match event { // Someone is asking for our sync From ad96bbdf8b38e7703ec5af62da44e1e0294c77ee Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:03:12 +0500 Subject: [PATCH 05/62] fix(zk): validate canonical DKG H and N shapes --- .../src/circuits/aggregation/node_dkg_fold.rs | 135 ++++++++++++++---- 1 file changed, 109 insertions(+), 26 deletions(-) diff --git a/crates/zk-prover/src/circuits/aggregation/node_dkg_fold.rs b/crates/zk-prover/src/circuits/aggregation/node_dkg_fold.rs index 3b2f3ac095..9b321a6769 100644 --- a/crates/zk-prover/src/circuits/aggregation/node_dkg_fold.rs +++ b/crates/zk-prover/src/circuits/aggregation/node_dkg_fold.rs @@ -21,6 +21,7 @@ use e3_events::{CircuitName, CircuitVariant, Proof}; use e3_fhe_params::BfvPreset; use e3_zk_helpers::CiphernodesCommitteeSize; use serde::Serialize; +use std::collections::HashSet; use std::time::Instant; fn proof_field_strings(proof: &Proof) -> Result, ZkError> { @@ -378,6 +379,53 @@ pub struct DkgAggregationInput<'a> { pub committee_addresses: &'a [Address], } +fn validate_dkg_aggregation_shape( + node_fold_count: usize, + party_ids: &[u64], + committee_address_count: usize, + committee: CiphernodesCommitteeSize, +) -> Result<(), ZkError> { + let expected = committee.values(); + if node_fold_count != party_ids.len() { + return Err(ZkError::InvalidInput( + "node_fold_proofs and party_ids length mismatch".into(), + )); + } + if node_fold_count != expected.h { + return Err(ZkError::InvalidInput(format!( + "DkgAggregator requires H={} honest NodeFold proofs for committee {}, got {}", + expected.h, committee, node_fold_count + ))); + } + if committee_address_count != expected.n { + return Err(ZkError::InvalidInput(format!( + "DkgAggregator requires N={} full committee addresses for committee {}, got {}", + expected.n, committee, committee_address_count + ))); + } + + let mut seen = HashSet::with_capacity(party_ids.len()); + for &party_id in party_ids { + let party_index = usize::try_from(party_id).map_err(|_| { + ZkError::InvalidInput(format!( + "DkgAggregator party id {party_id} does not fit a committee index" + )) + })?; + if party_index >= expected.n { + return Err(ZkError::InvalidInput(format!( + "DkgAggregator party id {party_id} is outside committee N={}", + expected.n + ))); + } + if !seen.insert(party_id) { + return Err(ZkError::InvalidInput(format!( + "DkgAggregator party id {party_id} is duplicated" + ))); + } + } + Ok(()) +} + #[derive(Serialize)] struct DkgAggregatorWitness { nodes_fold_vk: Vec, @@ -403,35 +451,15 @@ pub fn prove_dkg_aggregation( preset: BfvPreset, committee: CiphernodesCommitteeSize, ) -> Result { + validate_dkg_aggregation_shape( + input.node_fold_proofs.len(), + input.party_ids, + input.committee_addresses.len(), + committee, + )?; let artifacts_dir = prover.resolve_artifacts_dir(preset, committee.as_str()); let artifacts_dir = artifacts_dir.as_str(); - if input.node_fold_proofs.len() != input.party_ids.len() { - return Err(ZkError::InvalidInput( - "node_fold_proofs and party_ids length mismatch".into(), - )); - } - if input.node_fold_proofs.is_empty() { - return Err(ZkError::InvalidInput( - "prove_dkg_aggregation: need at least one NodeFold proof".into(), - )); - } let h = input.node_fold_proofs.len(); - // Full on-chain `topNodes` (must match compiled `N_PARTIES` in the circuit artifact). - // Do not use `preset.metadata().num_parties` — that is BFV search metadata, not circuit size. - let n_registered = input.committee_addresses.len(); - if n_registered == 0 { - return Err(ZkError::InvalidInput( - "prove_dkg_aggregation: committee_addresses must be non-empty (on-chain topNodes)" - .into(), - )); - } - #[cfg(debug_assertions)] - { - debug_assert_eq!( - h, n_registered, - "DkgAggregator honest-set H must equal registered committee size until expulsion enables H < N" - ); - } let nodes_fold_proof = if let Some(precomputed) = input.nodes_fold_proof { precomputed.clone() } else { @@ -610,3 +638,58 @@ pub fn prove_decryption_aggregation_jobs( } Ok(out) } + +#[cfg(test)] +mod tests { + use super::validate_dkg_aggregation_shape; + use e3_zk_helpers::CiphernodesCommitteeSize; + + #[test] + fn dkg_aggregation_accepts_all_canonical_h_of_n_shapes() { + for (committee, h, n) in [ + (CiphernodesCommitteeSize::Minimum, 2, 3), + (CiphernodesCommitteeSize::Micro, 5, 9), + (CiphernodesCommitteeSize::Small, 10, 19), + ] { + let party_ids: Vec = (0..h as u64).collect(); + validate_dkg_aggregation_shape(h, &party_ids, n, committee) + .expect("canonical committee must have H honest proofs and N addresses"); + } + } + + #[test] + fn dkg_aggregation_rejects_h_equal_to_n_for_minimum_committee() { + let error = + validate_dkg_aggregation_shape(3, &[0, 1, 2], 3, CiphernodesCommitteeSize::Minimum) + .unwrap_err(); + assert!(error.to_string().contains("requires H=2")); + } + + #[test] + fn dkg_aggregation_rejects_incomplete_honest_set() { + let error = validate_dkg_aggregation_shape(1, &[0], 3, CiphernodesCommitteeSize::Minimum) + .unwrap_err(); + assert!(error.to_string().contains("requires H=2")); + } + + #[test] + fn dkg_aggregation_rejects_wrong_full_committee_size() { + let error = + validate_dkg_aggregation_shape(2, &[0, 1], 2, CiphernodesCommitteeSize::Minimum) + .unwrap_err(); + assert!(error.to_string().contains("requires N=3")); + } + + #[test] + fn dkg_aggregation_rejects_duplicate_or_out_of_range_party_ids() { + let duplicate = + validate_dkg_aggregation_shape(2, &[1, 1], 3, CiphernodesCommitteeSize::Minimum) + .unwrap_err(); + assert!(duplicate.to_string().contains("duplicated")); + + let out_of_range = + validate_dkg_aggregation_shape(2, &[0, 3], 3, CiphernodesCommitteeSize::Minimum) + .unwrap_err(); + assert!(out_of_range.to_string().contains("outside committee N=3")); + } +} From e7f400ea98e0137fd4adf1c93ecf57eb1f9c8cc6 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:03:22 +0500 Subject: [PATCH 06/62] fix(sdk): subscribe to finalized sortition event --- packages/interfold-sdk/src/events/types.ts | 2 +- packages/interfold-sdk/tests/events.test.ts | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 packages/interfold-sdk/tests/events.test.ts diff --git a/packages/interfold-sdk/src/events/types.ts b/packages/interfold-sdk/src/events/types.ts index b2d70d5fee..add662a215 100644 --- a/packages/interfold-sdk/src/events/types.ts +++ b/packages/interfold-sdk/src/events/types.ts @@ -23,7 +23,7 @@ export enum InterfoldEventType { export enum RegistryEventType { COMMITTEE_REQUESTED = 'CommitteeRequested', COMMITTEE_PUBLISHED = 'CommitteePublished', - COMMITTEE_FINALIZED = 'CommitteeFinalized', + COMMITTEE_FINALIZED = 'SortitionCommitteeFinalized', INTERFOLD_SET = 'InterfoldSet', OWNERSHIP_TRANSFERRED = 'OwnershipTransferred', INITIALIZED = 'Initialized', diff --git a/packages/interfold-sdk/tests/events.test.ts b/packages/interfold-sdk/tests/events.test.ts new file mode 100644 index 0000000000..f800d675b8 --- /dev/null +++ b/packages/interfold-sdk/tests/events.test.ts @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +import { CiphernodeRegistryOwnable__factory } from '@interfold/contracts/types' +import { describe, expect, it } from 'vitest' + +import { RegistryEventType } from '../src/events/types' + +describe('RegistryEventType', () => { + it('uses the finalized-committee event name exposed by the registry ABI', () => { + const registryEventNames = CiphernodeRegistryOwnable__factory.abi.filter((item) => item.type === 'event').map((item) => item.name) + + expect(RegistryEventType.COMMITTEE_FINALIZED).toBe('SortitionCommitteeFinalized') + expect(registryEventNames).toContain(RegistryEventType.COMMITTEE_FINALIZED) + }) +}) From b858f367a789531602210c8517be9c1ab2d91a56 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:07:46 +0500 Subject: [PATCH 07/62] fix(startup): enforce a configurable readiness deadline --- crates/config/src/app_config.rs | 65 ++++++++++++++++++++++++++++ crates/entrypoint/src/start/start.rs | 55 +++++++++++++++++++++-- 2 files changed, 117 insertions(+), 3 deletions(-) diff --git a/crates/config/src/app_config.rs b/crates/config/src/app_config.rs index bfd45055c9..ebd613c2f3 100644 --- a/crates/config/src/app_config.rs +++ b/crates/config/src/app_config.rs @@ -60,12 +60,20 @@ pub struct NodeDefinition { /// Max concurrent CPU-bound jobs (ZK proofs + TrBFV). When unset, defaults to all CPUs minus /// `multithread_reserve_threads`. Override with env `E3_NODE__MULTITHREAD_CONCURRENT_JOBS`. pub multithread_concurrent_jobs: Option, + /// Hard deadline for construction and initial synchronization. A node that cannot reach live + /// protocol operation before this deadline exits non-zero instead of remaining falsely alive. + #[serde(default = "default_startup_timeout_secs")] + pub startup_timeout_secs: u64, } fn default_multithread_reserve_threads() -> usize { 1 } +fn default_startup_timeout_secs() -> u64 { + 30 * 60 +} + impl Default for NodeDefinition { fn default() -> Self { Self { @@ -84,6 +92,7 @@ impl Default for NodeDefinition { dashboard_port: None, multithread_reserve_threads: default_multithread_reserve_threads(), multithread_concurrent_jobs: None, + startup_timeout_secs: default_startup_timeout_secs(), } } } @@ -168,6 +177,9 @@ impl AppConfig { }; let node = node.clone(); + if node.startup_timeout_secs == 0 { + bail!("node.startup_timeout_secs must be greater than zero"); + } let config_dir_override = (node.config_dir != PathBuf::new()) .then_some(&node.config_dir) @@ -352,6 +364,11 @@ impl AppConfig { pub fn multithread_concurrent_jobs(&self) -> Option { self.node_def().multithread_concurrent_jobs } + + /// Maximum time allowed for construction and initial synchronization. + pub fn startup_timeout_secs(&self) -> u64 { + self.node_def().startup_timeout_secs + } } #[derive(Debug, Default, Deserialize, Serialize)] @@ -745,6 +762,54 @@ node: Ok(()) } + #[test] + fn test_startup_timeout_config_and_default() -> Result<()> { + let configured: UnscopedAppConfig = serde_yaml::from_str( + r#" +node: + startup_timeout_secs: 42 +"#, + )?; + let configured = configured.into_scoped_with_defaults( + "_default", + &PathBuf::from("/default/data"), + &PathBuf::from("/default/config"), + &PathBuf::from("/my/cwd"), + )?; + assert_eq!(configured.startup_timeout_secs(), 42); + + let default = UnscopedAppConfig::default().into_scoped_with_defaults( + "_default", + &PathBuf::from("/default/data"), + &PathBuf::from("/default/config"), + &PathBuf::from("/my/cwd"), + )?; + assert_eq!(default.startup_timeout_secs(), 30 * 60); + Ok(()) + } + + #[test] + fn test_zero_startup_timeout_is_rejected() -> Result<()> { + let unscoped: UnscopedAppConfig = serde_yaml::from_str( + r#" +node: + startup_timeout_secs: 0 +"#, + )?; + let error = unscoped + .into_scoped_with_defaults( + "_default", + &PathBuf::from("/default/data"), + &PathBuf::from("/default/config"), + &PathBuf::from("/my/cwd"), + ) + .expect_err("zero startup timeout must fail configuration"); + assert!(error + .to_string() + .contains("startup_timeout_secs must be greater than zero")); + Ok(()) + } + #[test] fn test_config_env_vars() { Jail::expect_with(|jail| { diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 15499c2099..979654205c 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -11,9 +11,20 @@ use e3_crypto::Cipher; use e3_zk_prover::ZkBackend; use rand::SeedableRng; use rand_chacha::ChaCha20Rng; +use std::future::Future; use std::sync::{Arc, Mutex}; +use std::time::Duration; use tracing::{info, instrument}; +async fn await_startup(future: F, timeout: Duration) -> Result +where + F: Future>, +{ + tokio::time::timeout(timeout, future) + .await + .with_context(|| format!("ciphernode startup did not complete within {timeout:?}"))? +} + #[instrument(name = "app", skip_all)] pub async fn execute(config: &AppConfig) -> Result { let rng = Arc::new(Mutex::new( @@ -31,7 +42,13 @@ pub async fn execute(config: &AppConfig) -> Result { .unwrap_or_else(|| "auto (CPUs - reserve)".to_string()) ); - let node = CiphernodeBuilder::new(rng.clone(), cipher.clone()) + let startup_timeout = Duration::from_secs(config.startup_timeout_secs()); + info!( + startup_timeout_secs = startup_timeout.as_secs(), + "Ciphernode startup deadline configured" + ); + + let build = CiphernodeBuilder::new(rng.clone(), cipher.clone()) .with_name(&config.name()) .with_logging() .with_persistence(&config.log_file(), &config.db_file()) @@ -49,8 +66,40 @@ pub async fn execute(config: &AppConfig) -> Result { .with_net(config.peers(), config.quic_port()) .with_shared_store() .with_shared_eventstore() - .build() - .await?; + .build(); + let node = await_startup(build, startup_timeout).await?; Ok(node) } + +#[cfg(test)] +mod tests { + use super::await_startup; + use anyhow::{bail, Result}; + use std::time::Duration; + + #[tokio::test] + async fn startup_deadline_returns_completed_result() -> Result<()> { + let value = + await_startup(async { Ok::<_, anyhow::Error>(7) }, Duration::from_secs(1)).await?; + assert_eq!(value, 7); + Ok(()) + } + + #[tokio::test] + async fn startup_deadline_fails_instead_of_waiting_forever() -> Result<()> { + let error = await_startup( + std::future::pending::>(), + Duration::from_millis(5), + ) + .await + .expect_err("pending startup must hit its deadline"); + if !error + .to_string() + .contains("startup did not complete within 5ms") + { + bail!("unexpected timeout error: {error:#}"); + } + Ok(()) + } +} From 3e092e0619af2fb4a53bf2d31df676f24237692a Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:14:11 +0500 Subject: [PATCH 08/62] fix(evm): serialize nonce allocation across writers --- .../evm/src/actors/ciphernode_registry_sol.rs | 17 +++-- crates/evm/src/actors/interfold_sol_writer.rs | 28 +++++--- .../src/actors/slashing_manager_sol_writer.rs | 11 ++- crates/evm/src/helpers.rs | 70 ++++++++++++++++++- 4 files changed, 102 insertions(+), 24 deletions(-) diff --git a/crates/evm/src/actors/ciphernode_registry_sol.rs b/crates/evm/src/actors/ciphernode_registry_sol.rs index 45fe714f55..64f59a2443 100644 --- a/crates/evm/src/actors/ciphernode_registry_sol.rs +++ b/crates/evm/src/actors/ciphernode_registry_sol.rs @@ -8,7 +8,7 @@ use crate::actors::evm_parser::EvmParser; use crate::contracts::ICiphernodeRegistry; use crate::domain::ciphernode_registry_events::extractor; use crate::domain::error_decoder::{decode_error_from_str, format_evm_error}; -use crate::helpers::{encode_zk_proof, send_tx_with_retry, EthProvider}; +use crate::helpers::{encode_zk_proof, send_tx_with_retry, transaction_nonce_guard, EthProvider}; use crate::messages::{EvmEventProcessor, InterfoldEvmEvent}; use actix::prelude::*; use alloy::{ @@ -365,6 +365,7 @@ pub async fn submit_ticket_to_registry( ) -> Result { let e3_id: U256 = e3_id.try_into()?; - let from_address = provider.provider().default_signer_address(); - let current_nonce = provider - .provider() - .get_transaction_count(from_address) - .pending() - .await?; - // `None` => proof aggregation disabled; contract accepts empty bytes in that case. let proof: Bytes = match decryption_aggregator_proof { Some(p) => encode_zk_proof(p)?, @@ -332,13 +325,23 @@ async fn publish_plaintext_output( info!("publishPlaintextOutput() e3_id={:?}", e3_id); let decrypted_output = Bytes::from(decrypted_output.clone()); let proof = proof.clone(); - let contract = IInterfold::new(contract_address, provider.provider()); + let provider = provider.clone(); async move { + let _nonce_guard = transaction_nonce_guard(&provider).await; + let from_address = provider.provider().default_signer_address(); + let current_nonce = provider + .provider() + .get_transaction_count(from_address) + .pending() + .await?; + let contract = IInterfold::new(contract_address, provider.provider()); let builder = contract .publishPlaintextOutput(e3_id, decrypted_output, proof) .nonce(current_nonce); - let receipt = builder.send().await?.get_receipt().await?; + let pending = builder.send().await?; + drop(_nonce_guard); + let receipt = pending.get_receipt().await?; Ok(receipt) } }, @@ -366,6 +369,7 @@ async fn process_e3_failure( info!("processE3Failure() e3_id={:?}", e3_id); + let _nonce_guard = transaction_nonce_guard(&provider).await; let from_address = provider.provider().default_signer_address(); let current_nonce = provider .provider() @@ -374,6 +378,8 @@ async fn process_e3_failure( .await?; let contract = IInterfold::new(contract_address, provider.provider()); let builder = contract.processE3Failure(e3_id).nonce(current_nonce); - let receipt = builder.send().await?.get_receipt().await?; + let pending = builder.send().await?; + drop(_nonce_guard); + let receipt = pending.get_receipt().await?; Ok(receipt) } diff --git a/crates/evm/src/actors/slashing_manager_sol_writer.rs b/crates/evm/src/actors/slashing_manager_sol_writer.rs index d4ff31dc25..833d9f977c 100644 --- a/crates/evm/src/actors/slashing_manager_sol_writer.rs +++ b/crates/evm/src/actors/slashing_manager_sol_writer.rs @@ -13,7 +13,7 @@ use crate::contracts::{ICiphernodeRegistry, ISlashingManager}; use crate::domain::attestation_evidence::encode_attestation_evidence; use crate::domain::error_decoder::format_evm_error; use crate::domain::slash_submission::{should_submit_slash, submission_delay, submission_rank}; -use crate::helpers::EthProvider; +use crate::helpers::{transaction_nonce_guard, EthProvider}; use crate::send_tx_with_retry; use actix::prelude::*; use actix::Addr; @@ -218,6 +218,7 @@ async fn submit_slash_proposal( let provider = provider.clone(); async move { + let _nonce_guard = transaction_nonce_guard(&provider).await; let from_address = provider.provider().default_signer_address(); let current_nonce = provider .provider() @@ -225,23 +226,21 @@ async fn submit_slash_proposal( .pending() .await?; let contract = ISlashingManager::new(contract_address, provider.provider()); - let receipt = if let Some(pid) = party_id { + let pending = if let Some(pid) = party_id { contract .proposeSlashByDkgParty(e3_id, pid, proof) .nonce(current_nonce) .send() .await? - .get_receipt() - .await? } else { contract .proposeSlash(e3_id, operator, proof) .nonce(current_nonce) .send() .await? - .get_receipt() - .await? }; + drop(_nonce_guard); + let receipt = pending.get_receipt().await?; Ok(receipt) } }) diff --git a/crates/evm/src/helpers.rs b/crates/evm/src/helpers.rs index aec67b98af..4e7675eaa9 100644 --- a/crates/evm/src/helpers.rs +++ b/crates/evm/src/helpers.rs @@ -12,7 +12,7 @@ use alloy::{ BlobGasFiller, ChainIdFiller, FillProvider, GasFiller, JoinFill, NonceFiller, SimpleNonceManager, WalletFiller, }, - Identity, Provider, ProviderBuilder, RootProvider, + Identity, Provider, ProviderBuilder, RootProvider, WalletProvider, }, rpc::types::TransactionReceipt, signers::local::PrivateKeySigner, @@ -28,7 +28,10 @@ use alloy::{ Authorization, }, }; -use alloy::{primitives::Bytes, sol_types::SolValue}; +use alloy::{ + primitives::{Address, Bytes}, + sol_types::SolValue, +}; use anyhow::{Context, Result}; use base64::{engine::general_purpose::STANDARD, Engine}; use e3_config::{RpcAuth, RPC}; @@ -36,7 +39,14 @@ use e3_crypto::Cipher; use e3_data::Repository; use e3_events::Proof; use e3_utils::{retry_with_backoff, RetryError}; -use std::{env, future::Future, pin::Pin, sync::Arc}; +use std::{ + collections::HashMap, + env, + future::Future, + pin::Pin, + sync::{Arc, Mutex as StdMutex, OnceLock}, +}; +use tokio::sync::{Mutex as AsyncMutex, OwnedMutexGuard}; use tracing::info; use zeroize::{Zeroize, Zeroizing}; @@ -115,6 +125,34 @@ impl EthProvider

{ } } +type NonceLockKey = (u64, Address); + +fn nonce_lock(chain_id: u64, signer: Address) -> Arc> { + static LOCKS: OnceLock>>>> = OnceLock::new(); + let locks = LOCKS.get_or_init(|| StdMutex::new(HashMap::new())); + let mut locks = locks + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + Arc::clone( + locks + .entry((chain_id, signer)) + .or_insert_with(|| Arc::new(AsyncMutex::new(()))), + ) +} + +/// Serialize nonce lookup and submission for one signer on one chain. +/// +/// Independent writer actors share a provider signer and previously raced by reading the same +/// pending nonce. Writers release the guard after the RPC accepts the transaction, before waiting +/// for a receipt, so a slow confirmation cannot block unrelated submissions indefinitely. +pub(crate) async fn transaction_nonce_guard

(provider: &EthProvider

) -> OwnedMutexGuard<()> +where + P: Provider + WalletProvider + Clone, +{ + let signer = provider.provider().default_signer_address(); + nonce_lock(provider.chain_id(), signer).lock_owned().await +} + pub type ProviderFactory

= Arc Pin>> + Send>> + Send + Sync>; @@ -319,6 +357,7 @@ mod tests { use alloy_dyn_abi::DynSolType; use e3_events::{CircuitName, Proof}; use e3_utils::ArcBytes; + use std::sync::atomic::{AtomicUsize, Ordering}; /// Verifies encode_zk_proof produces ABI: abi.decode(proof, (bytes, bytes32[])) #[test] @@ -390,4 +429,29 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn nonce_lock_serializes_one_signer_per_chain() { + let key = (9_876_543, Address::repeat_byte(0x42)); + let active = Arc::new(AtomicUsize::new(0)); + let maximum = Arc::new(AtomicUsize::new(0)); + let mut tasks = Vec::new(); + + for _ in 0..32 { + let active = Arc::clone(&active); + let maximum = Arc::clone(&maximum); + tasks.push(tokio::spawn(async move { + let _guard = nonce_lock(key.0, key.1).lock_owned().await; + let now = active.fetch_add(1, Ordering::SeqCst) + 1; + maximum.fetch_max(now, Ordering::SeqCst); + tokio::task::yield_now().await; + active.fetch_sub(1, Ordering::SeqCst); + })); + } + + for task in tasks { + task.await.expect("nonce-lock task must not panic"); + } + assert_eq!(maximum.load(Ordering::SeqCst), 1); + } } From 6b0a3075d3e70bd0e7611d9771a408f70dffe994 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:15:41 +0500 Subject: [PATCH 09/62] fix(events): replace probabilistic dedup with bounded exact IDs --- Cargo.lock | 1 - crates/events/Cargo.toml | 1 - crates/events/src/eventbus.rs | 221 +++++++++++++++++++++++-- crates/events/src/eventstore.rs | 85 +++++----- crates/sync/src/actors/sync.rs | 2 +- crates/sync/src/domain/sync_planner.rs | 2 +- 6 files changed, 257 insertions(+), 55 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eae191c4a3..0cd7ffc6cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3373,7 +3373,6 @@ dependencies = [ "alloy-sol-types", "anyhow", "bincode 1.3.3", - "bloom", "bs58", "chrono", "derivative", diff --git a/crates/events/Cargo.toml b/crates/events/Cargo.toml index b1c4984437..9a8ba51778 100644 --- a/crates/events/Cargo.toml +++ b/crates/events/Cargo.toml @@ -13,7 +13,6 @@ alloy-primitives = { workspace = true } alloy-sol-types = { workspace = true } anyhow = { workspace = true } bincode = { workspace = true } -bloom = { workspace = true } bs58 = { workspace = true } chrono = { workspace = true } derivative = { workspace = true } diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index 5e5cc018c3..bdd6ce71d1 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -7,10 +7,10 @@ use crate::traits::{ErrorEvent, Event}; use crate::EventType; use actix::prelude::*; -use bloom::{BloomFilter, ASMS}; use e3_utils::{colorize, Color, MAILBOX_LIMIT, MAILBOX_LIMIT_LARGE}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; +use std::hash::Hash; use std::marker::PhantomData; use std::time::Duration; use tokio::sync::mpsc; @@ -30,10 +30,61 @@ impl Default for EventBusConfig { } } -fn default_bloomfilter() -> BloomFilter { - let num_items = 10000000; - let fp_rate = 0.001; - BloomFilter::with_rate(fp_rate, num_items) +/// Number of recently observed event IDs retained for exact deduplication. +/// +/// This bounds memory while retaining a large replay window. Once full, IDs are +/// evicted in first-observed order and a later occurrence is accepted again. +const DEFAULT_DEDUP_CAPACITY: usize = 250_000; + +/// A bounded, exact FIFO set. +/// +/// `HashSet` uses equality to resolve hash collisions, so two distinct event IDs +/// can never be treated as duplicates solely because they share a hash. Duplicate +/// observations do not refresh an ID's position, keeping eviction deterministic +/// and preventing duplicate traffic from growing the queue. +struct ExactDedup { + capacity: usize, + ids: HashSet, + insertion_order: VecDeque, +} + +impl ExactDedup +where + I: Clone + Eq + Hash, +{ + fn new(capacity: usize) -> Self { + Self { + capacity, + ids: HashSet::new(), + insertion_order: VecDeque::new(), + } + } + + fn contains(&self, id: &I) -> bool { + self.ids.contains(id) + } + + fn insert(&mut self, id: I) { + if self.capacity == 0 || self.ids.contains(&id) { + return; + } + + if self.ids.len() == self.capacity { + let removed = self + .insertion_order + .pop_front() + .is_some_and(|oldest| self.ids.remove(&oldest)); + if !removed { + tracing::error!("exact dedup state diverged; resetting the bounded window"); + self.ids.clear(); + self.insertion_order.clear(); + } + } + + if self.ids.insert(id.clone()) { + self.insertion_order.push_back(id); + } + } } ////////////////////////////////////////////////////////////////////////////// @@ -46,7 +97,7 @@ fn default_bloomfilter() -> BloomFilter { /// being published. pub struct EventBus { config: EventBusConfig, - ids: BloomFilter, + ids: ExactDedup, listeners: HashMap>>, } @@ -59,10 +110,14 @@ impl Actor for EventBus { impl EventBus { pub fn new(config: EventBusConfig) -> Self { + Self::with_dedup_capacity(config, DEFAULT_DEDUP_CAPACITY) + } + + fn with_dedup_capacity(config: EventBusConfig, dedup_capacity: usize) -> Self { EventBus { config, listeners: HashMap::new(), - ids: default_bloomfilter(), + ids: ExactDedup::new(dedup_capacity), } } @@ -98,12 +153,14 @@ impl EventBus { source.do_send(Subscribe::new(EventType::All, filter.recipient())); } - fn track(&mut self, event: E) { - self.ids.insert(&event.event_id()); + fn track(&mut self, event: &E) { + if self.config.deduplicate { + self.ids.insert(event.event_id()); + } } fn is_duplicate(&self, event: &E) -> bool { - self.ids.contains(&event.event_id()) + self.config.deduplicate && self.ids.contains(&event.event_id()) } } @@ -112,7 +169,7 @@ impl Default for EventBus { Self { config: EventBusConfig::default(), listeners: HashMap::new(), - ids: default_bloomfilter(), + ids: ExactDedup::new(DEFAULT_DEDUP_CAPACITY), } } } @@ -137,7 +194,7 @@ impl Handler for EventBus { } tracing::info!("{} {}", colorize(">>>", Color::Yellow), event); - self.track(event); + self.track(&event); } } @@ -422,3 +479,141 @@ impl Handler> for HistoryCollector { self.history.clone() } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::{AggregateId, WithAggregateId}; + use std::hash::{Hash, Hasher}; + + #[derive(Clone, Debug, Eq, PartialEq)] + struct CollidingId(u64); + + impl Hash for CollidingId { + fn hash(&self, state: &mut H) { + // Deliberately give every ID the same hash. Exact deduplication must + // still distinguish IDs by equality. + 0_u8.hash(state); + } + } + + impl fmt::Display for CollidingId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "collision:{}", self.0) + } + } + + #[derive(Clone, Debug)] + struct TestData; + + impl WithAggregateId for TestData { + fn get_aggregate_id(&self) -> AggregateId { + AggregateId::new(0) + } + } + + #[derive(Clone, Debug, Message)] + #[rtype(result = "()")] + struct CollidingEvent { + id: CollidingId, + data: TestData, + } + + impl CollidingEvent { + fn new(id: u64) -> Self { + Self { + id: CollidingId(id), + data: TestData, + } + } + } + + impl fmt::Display for CollidingEvent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "CollidingEvent({})", self.id) + } + } + + impl Event for CollidingEvent { + type Id = CollidingId; + type Data = TestData; + + fn event_id(&self) -> Self::Id { + self.id.clone() + } + + fn event_type(&self) -> String { + "CollidingEvent".to_owned() + } + + fn get_data(&self) -> &Self::Data { + &self.data + } + + fn into_data(self) -> Self::Data { + self.data + } + } + + #[actix::test] + async fn distinct_ids_with_identical_hashes_are_both_delivered() -> anyhow::Result<()> { + let bus = EventBus::with_dedup_capacity(EventBusConfig::default(), 4).start(); + let history = EventBus::history(&bus); + + bus.send(CollidingEvent::new(1)).await?; + bus.send(CollidingEvent::new(2)).await?; + + let received = history.send(TakeEvents::new(2)).await?; + assert!(!received.timed_out); + assert_eq!( + received + .events + .into_iter() + .map(|event| event.id.0) + .collect::>(), + vec![1, 2] + ); + Ok(()) + } + + #[actix::test] + async fn fifo_eviction_is_deterministic_and_evicted_ids_are_reaccepted() -> anyhow::Result<()> { + let bus = EventBus::with_dedup_capacity(EventBusConfig::default(), 2).start(); + let history = EventBus::history(&bus); + + bus.send(CollidingEvent::new(1)).await?; + bus.send(CollidingEvent::new(2)).await?; + bus.send(CollidingEvent::new(1)).await?; // duplicate; does not refresh FIFO position + bus.send(CollidingEvent::new(3)).await?; // evicts 1 + bus.send(CollidingEvent::new(2)).await?; // duplicate; 2 is still retained + bus.send(CollidingEvent::new(1)).await?; // reaccepted; evicts 2 + bus.send(CollidingEvent::new(2)).await?; // reaccepted + + let received = history.send(TakeEvents::new(5)).await?; + assert!(!received.timed_out); + assert_eq!( + received + .events + .into_iter() + .map(|event| event.id.0) + .collect::>(), + vec![1, 2, 3, 1, 2] + ); + Ok(()) + } + + #[actix::test] + async fn disabled_deduplication_delivers_repeated_ids() -> anyhow::Result<()> { + let config = EventBusConfig { deduplicate: false }; + let bus = EventBus::with_dedup_capacity(config, 2).start(); + let history = EventBus::history(&bus); + + bus.send(CollidingEvent::new(1)).await?; + bus.send(CollidingEvent::new(1)).await?; + + let received = history.send(TakeEvents::new(2)).await?; + assert!(!received.timed_out); + assert_eq!(received.events.len(), 2); + Ok(()) + } +} diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 9f6e9175b5..01e9694775 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -6,39 +6,59 @@ use crate::{ events::{StoreEventRequested, StoreEventResponse}, - EventContextAccessors, EventLog, EventStoreFilter, EventStoreQueryBy, EventStoreQueryResponse, - InterfoldEvent, Seq, SequenceIndex, Sequenced, Ts, Unsequenced, + Event, EventContextAccessors, EventLog, EventStoreFilter, EventStoreQueryBy, + EventStoreQueryResponse, InterfoldEvent, Seq, SequenceIndex, Sequenced, Ts, Unsequenced, }; use actix::{Actor, Handler}; use anyhow::{bail, Result}; use tracing::{error, warn}; const INDEX_RECONCILE_PAGE_SIZE: usize = 1_024; -const MAX_STORAGE_ERRORS: u64 = 10; pub struct EventStore { index: I, log: L, - storage_errors: u64, } impl EventStore { /// Attempt to store an event. Returns the sequenced event on success, - /// `None` if the event was a duplicate, or an error on failure. + /// `None` if the complete event is an exact duplicate, or an error on failure. pub fn store_event( &mut self, event: InterfoldEvent, ) -> Result>> { let ts = event.ts(); - if self.index.get(ts)?.is_some() { - warn!("Event already stored at timestamp {ts}! This might happen when recovering from a snapshot. Skipping storage"); - self.storage_errors += 1; - if self.storage_errors > MAX_STORAGE_ERRORS { + if let Some(indexed_seq) = self.index.get(ts)? { + let Some((logged_seq, existing)) = self.log.read_from_bounded(indexed_seq, 1).next() + else { bail!( - "The eventstore had too many storage errors! {}", - self.storage_errors + "event index corruption at timestamp {ts}: sequence {indexed_seq} is missing \ + from the event log" ); + }; + if logged_seq != indexed_seq { + bail!( + "event index corruption at timestamp {ts}: requested sequence {indexed_seq}, \ + log returned sequence {logged_seq}" + ); + } + if existing == event { + warn!( + timestamp = ts, + sequence = indexed_seq, + event_id = %event.id(), + event_type = %event.get_data().event_type(), + "Ignoring exact duplicate event" + ); + return Ok(None); } - return Ok(None); + bail!( + "event timestamp collision at {ts}, sequence {indexed_seq}: existing {} ({}) \ + conflicts with incoming {} ({})", + existing.id(), + existing.get_data().event_type(), + event.id(), + event.get_data().event_type() + ); } let seq = self.log.append(&event)?; self.index.insert(ts, seq)?; @@ -119,11 +139,7 @@ impl EventStore { impl EventStore { pub fn new(index: I, log: L) -> Self { - let mut store = Self { - index, - log, - storage_errors: 0, - }; + let mut store = Self { index, log }; store.reconcile_index(); store } @@ -503,36 +519,29 @@ mod tests { } #[test] - fn store_event_returns_none_for_duplicate_timestamp() { + fn exact_duplicate_is_indefinitely_idempotent() { let mut store = new_store(); - store.store_event(make_local_event(100)).unwrap(); + let event = make_local_event(100); + store.store_event(event.clone()).unwrap(); - let result = store.store_event(make_local_event(100)).unwrap(); + for _ in 0..100 { + let result = store.store_event(event.clone()).unwrap(); + assert!(result.is_none()); + } - assert!(result.is_none()); - assert_eq!(store.storage_errors, 1); - // Log should still have only one event - assert_eq!(store.log.read_from(0).count(), 1); + assert_eq!(store.log.read_from(1).count(), 1); } #[test] - fn store_event_bails_after_max_storage_errors() { + fn conflicting_event_at_same_timestamp_fails_immediately() { let mut store = new_store(); store.store_event(make_local_event(100)).unwrap(); - for _ in 0..MAX_STORAGE_ERRORS { - let result = store.store_event(make_local_event(100)).unwrap(); - assert!(result.is_none()); - } - - assert_eq!(store.storage_errors, MAX_STORAGE_ERRORS); - - let result = store.store_event(make_local_event(100)); - assert!(result.is_err()); - assert!(result - .unwrap_err() - .to_string() - .contains("too many storage errors")); + let error = store.store_event(make_network_event(100)).unwrap_err(); + let message = error.to_string(); + assert!(message.contains("timestamp collision"), "{message}"); + assert!(message.contains("sequence 1"), "{message}"); + assert_eq!(store.log.read_from(1).count(), 1); } // =========================================================================== diff --git a/crates/sync/src/actors/sync.rs b/crates/sync/src/actors/sync.rs index 43d9677f10..3f90fe1e66 100644 --- a/crates/sync/src/actors/sync.rs +++ b/crates/sync/src/actors/sync.rs @@ -74,7 +74,7 @@ pub async fn sync( // 4. Replay the EventStore events to all listeners (except effects). // Skip infrastructure events (SyncEnded, EffectsEnabled, HistoricalEvmSyncStart) because // they will be re-published by this sync process (steps 5, 8, 10). Replaying them here - // would poison the EventBus bloom-filter deduplication: the replayed event has the same + // would poison the EventBus deduplication window: the replayed event has the same // EventId (payload hash) as the one we publish later, causing the later event to be // silently dropped. This is critical for SyncEnded, if the EvmChainGateway never // receives it, the gateway stays in BufferUntilLive and all live EVM events are lost. diff --git a/crates/sync/src/domain/sync_planner.rs b/crates/sync/src/domain/sync_planner.rs index aa1b6a03cb..76844abafe 100644 --- a/crates/sync/src/domain/sync_planner.rs +++ b/crates/sync/src/domain/sync_planner.rs @@ -14,7 +14,7 @@ use std::collections::BTreeMap; /// /// Infrastructure events (`SyncEnded`, `EffectsEnabled`, `HistoricalEvmSyncStart`, /// `HistoricalNetSyncStart`) are re-published by the sync process itself, so replaying them -/// would poison the EventBus bloom-filter dedup. They must be skipped during replay. +/// would poison the EventBus deduplication window. They must be skipped during replay. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ReplayDecision { /// Forward the event to listeners. From bfa34c2d601c4221559a9de33003a33db5868bbb Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 13 Jul 2026 22:49:38 +0500 Subject: [PATCH 10/62] fix(net): bound and advance historical sync requests --- crates/net/src/actors/net_sync_manager.rs | 344 ++++++++++++++++++++-- crates/net/src/domain/net_event_batch.rs | 226 +++++++++++++- crates/net/src/domain/sync_coordinator.rs | 121 ++++++-- crates/net/src/events.rs | 3 + crates/net/src/net_interface.rs | 55 ++-- 5 files changed, 683 insertions(+), 66 deletions(-) diff --git a/crates/net/src/actors/net_sync_manager.rs b/crates/net/src/actors/net_sync_manager.rs index a7630f4b13..8186b6c7a3 100644 --- a/crates/net/src/actors/net_sync_manager.rs +++ b/crates/net/src/actors/net_sync_manager.rs @@ -13,6 +13,7 @@ use e3_events::{ NetReady, TsAgg, TypedEvent, Unsequenced, }; use e3_utils::MAILBOX_LIMIT; +use libp2p::PeerId; use serde::{Deserialize, Serialize}; use std::{ collections::{HashMap, HashSet}, @@ -28,10 +29,16 @@ use crate::{ direct_responder::DirectResponder, domain::{ build_sync_batch, - net_event_batch::{fetch_all_batched_events, FetchEventsSince}, + net_event_batch::{ + fetch_all_batched_events_with_budget, FetchEventsSince, SyncFetchBudget, + }, + sync_coordinator::sync_scan_limit, EventTranslationService, NetReadiness, ReadinessDecision, SyncBatchOutcome, }, - events::{await_event, GossipData, IncomingRequest, NetCommand, NetEvent, PeerTarget}, + events::{ + await_event, GossipData, IncomingRequest, NetCommand, NetEvent, PeerTarget, + ProtocolResponse, + }, }; /// Maximum time to wait for a `ConnectionEstablished` event after all dials @@ -49,6 +56,15 @@ const SYNC_RECOVERY_RETRY_INTERVAL: Duration = Duration::from_secs(15); /// Number of recovery rounds to try for failed aggregates after the initial fetch pass. const SYNC_RECOVERY_MAX_ATTEMPTS: usize = 3; +/// Bound remote work independently of the actor mailbox. Per-peer admission prevents one +/// authenticated transport identity from occupying the global allowance. +const MAX_IN_FLIGHT_SYNC_REQUESTS: usize = 16; +const MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER: usize = 2; + +/// Expire storage requests before libp2p's 30-second request timeout so a failed local query cannot +/// retain a responder and permanently consume one of the bounded in-flight slots. +const INCOMING_SYNC_REQUEST_TIMEOUT: Duration = Duration::from_secs(25); + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SyncResponseValue { pub events: Vec>, @@ -76,6 +92,11 @@ pub struct SyncRequestSucceeded { pub response: SyncResponseValue, } +struct PendingSyncRequest { + peer: PeerId, + responder: DirectResponder, +} + pub struct NetSyncManager { /// Interfold EventBus bus: BusHandle, @@ -84,7 +105,7 @@ pub struct NetSyncManager { /// NetEvents receiver to receive events rx: Arc>, eventstore: Recipient>, - requests: HashMap, + requests: HashMap, /// Pure readiness state machine. readiness: NetReadiness, /// Gossipsub topic used to re-broadcast our own forwardable artifacts after a restart. @@ -130,6 +151,43 @@ impl NetSyncManager { Ok(()) } + fn request_capacity_error(&self, peer: &PeerId) -> Option<&'static str> { + if self.requests.len() >= MAX_IN_FLIGHT_SYNC_REQUESTS { + return Some("too many in-flight sync requests"); + } + let peer_requests = self + .requests + .values() + .filter(|request| &request.peer == peer) + .count(); + if peer_requests >= MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER { + return Some("too many in-flight sync requests from this peer"); + } + None + } + + fn expire_sync_request(&mut self, id: CorrelationId) { + let Some(pending) = self.requests.remove(&id) else { + return; + }; + warn!( + peer = %pending.peer, + correlation_id = %id, + timeout_ms = INCOMING_SYNC_REQUEST_TIMEOUT.as_millis(), + "Incoming historical-sync storage query timed out" + ); + if let Err(error) = pending.responder.respond(ProtocolResponse::Error( + "historical sync request timed out".to_string(), + )) { + warn!( + peer = %pending.peer, + correlation_id = %id, + %error, + "Failed to send historical-sync timeout response" + ); + } + } + /// After a restart, proactively re-gossip this node's own already-produced forwardable DKG /// artifacts (H3/H11). Resume from a persisted phase is otherwise passive: the restored /// keyshare/aggregator actors wait for peer documents and never re-emit their own outputs, so @@ -137,7 +195,7 @@ impl NetSyncManager { /// node to its phase timeout. /// /// The artifacts are sent straight to libp2p as `GossipPublish`, bypassing both the EventBus - /// dedup bloom (which already tracked them during replay) and the translator (which is only + /// dedup window (which already tracked them during replay) and the translator (which is only /// created on `EffectsEnabled`). Re-broadcasting the byte-identical original payload is /// equivocation-safe (peers dedup by event id) and idempotent. The query is bounded to the /// snapshot-cursor window so only the in-flight (un-delivered) artifacts are re-sent. @@ -340,18 +398,58 @@ impl Handler> for NetSyncManager { impl Handler for NetSyncManager { type Result = (); fn handle(&mut self, msg: IncomingRequest, ctx: &mut Self::Context) -> Self::Result { - trap(EType::Net, &self.bus, || { + trap(EType::Net, &self.bus.clone(), || { + let IncomingRequest { peer, responder } = msg; + let fetch_request: FetchEventsSince = match responder.try_request_into() { + Ok(request) => request, + Err(error) => { + warn!(%peer, %error, "Rejecting malformed historical-sync request"); + responder.bad_request("malformed historical sync request")?; + return Ok(()); + } + }; + if fetch_request.limit() == 0 { + responder.bad_request("limit must be greater than 0")?; + return Ok(()); + } + if let Some(reason) = self.request_capacity_error(&peer) { + warn!( + %peer, + in_flight = self.requests.len(), + "Rejecting historical-sync request: {reason}" + ); + responder.bad_request(reason)?; + return Ok(()); + } + let id = CorrelationId::new(); - info!("Processing incoming request with correlation={}", id); - let fetch_request: FetchEventsSince = msg.responder.try_request_into()?; - self.requests.insert(id, msg.responder); + let scan_limit = sync_scan_limit(fetch_request.limit()); + info!( + peer = %peer, + correlation_id = %id, + requested_limit = fetch_request.limit(), + scan_limit, + "Processing incoming historical-sync request" + ); let query: HashMap = HashMap::from([(fetch_request.aggregate_id(), fetch_request.since())]); - self.eventstore.try_send(EventStoreQueryBy::::new( - id, - query, - ctx.address().recipient(), - ))?; + self.requests + .insert(id, PendingSyncRequest { peer, responder }); + let storage_query = + EventStoreQueryBy::::new(id, query, ctx.address().recipient()) + .with_limit(scan_limit as u64); + if let Err(error) = self.eventstore.try_send(storage_query) { + if let Some(pending) = self.requests.remove(&id) { + pending.responder.respond(ProtocolResponse::Error( + "historical sync storage unavailable".to_string(), + ))?; + } + warn!(%peer, correlation_id = %id, %error, "Failed to query EventStore for sync"); + return Ok(()); + } + ctx.run_later(INCOMING_SYNC_REQUEST_TIMEOUT, move |this, _| { + this.expire_sync_request(id); + }); Ok(()) }); } @@ -369,14 +467,14 @@ impl Handler for NetSyncManager { } trap(EType::Net, &self.bus.clone(), || { info!("Received response from eventstore."); - let Some(responder) = self.requests.remove(&msg.id()) else { + let Some(pending) = self.requests.remove(&msg.id()) else { bail!("responder not found for {}", msg.id()); }; - let fetch_request: FetchEventsSince = responder.try_request_into()?; + let fetch_request: FetchEventsSince = pending.responder.try_request_into()?; match build_sync_batch(msg.into_events(), &fetch_request) { - SyncBatchOutcome::BadRequest(reason) => responder.bad_request(reason)?, - SyncBatchOutcome::Batch(batch) => responder.ok(batch)?, + SyncBatchOutcome::BadRequest(reason) => pending.responder.bad_request(reason)?, + SyncBatchOutcome::Batch(batch) => pending.responder.ok(batch)?, } Ok(()) @@ -423,18 +521,20 @@ async fn fetch_historical_events_for_aggregate( net_events: &Arc>, aggregate_id: AggregateId, since: u128, + budget: &mut SyncFetchBudget, ) -> Result>> { let requester = DirectRequester::builder(net_cmds.clone(), net_events.clone()) .max_retries(SYNC_FETCH_MAX_RETRIES) .retry_timeout(SYNC_FETCH_RETRY_TIMEOUT) .build(); - fetch_all_batched_events::>( + fetch_all_batched_events_with_budget::>( requester, PeerTarget::Random, aggregate_id, since, 100, + budget, ) .await } @@ -486,14 +586,21 @@ async fn handle_sync_request_event( let mut all_events: Vec> = Vec::new(); let mut latest_timestamp: u128 = 0; let mut failed_aggregates: Vec = Vec::new(); + let mut budget = SyncFetchBudget::production(); for (aggregate_id, since) in event.since.iter() { info!( "Requesting batched events for aggregate_id={} since={}", aggregate_id, since ); - match fetch_historical_events_for_aggregate(&net_cmds, &net_events, *aggregate_id, *since) - .await + match fetch_historical_events_for_aggregate( + &net_cmds, + &net_events, + *aggregate_id, + *since, + &mut budget, + ) + .await { Ok(events) => { info!( @@ -510,6 +617,9 @@ async fn handle_sync_request_event( } } Err(e) => { + if budget.is_exhausted() { + return Err(e).context("historical net sync exhausted its global budget"); + } warn!( "Failed to fetch events for aggregate_id={}: {e}. Continuing with available events.", aggregate_id @@ -568,6 +678,7 @@ async fn handle_sync_request_event( &net_events, aggregate_id, since, + &mut budget, ) .await { @@ -587,6 +698,10 @@ async fn handle_sync_request_event( } } Err(e) => { + if budget.is_exhausted() { + return Err(e) + .context("historical net sync exhausted its global budget"); + } warn!( attempt = recovery_attempt, "Retry failed for aggregate_id={}: {e}", aggregate_id @@ -629,14 +744,17 @@ async fn handle_sync_request_event( #[cfg(test)] mod tests { use super::*; - use crate::events::NetCommand; + use crate::{ + direct_responder::ChannelType, + events::{IncomingRequest, NetCommand}, + }; use actix::{Actor, Context as ActixContext, Handler}; use e3_ciphernode_builder::EventSystem; use e3_events::{ E3id, EventSource, InterfoldEvent, PlaintextAggregated, TestEvent, Unsequenced, }; use e3_utils::ArcBytes; - use tokio::sync::{broadcast, mpsc}; + use tokio::sync::{broadcast, mpsc, mpsc::UnboundedSender}; /// Minimal EventStore stand-in so `NetSyncManager::new` can be constructed in tests; the /// re-broadcast unit test drives `handle_rebroadcast_response` directly and never queries it. @@ -649,6 +767,61 @@ mod tests { fn handle(&mut self, _: EventStoreQueryBy, _: &mut Self::Context) {} } + struct RecordingEventStore { + queries: UnboundedSender>, + } + + impl Actor for RecordingEventStore { + type Context = ActixContext; + } + + impl Handler> for RecordingEventStore { + type Result = (); + fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) { + let _ = self.queries.send(msg.limit()); + // Intentionally retain no response. Tests exercise the manager's in-flight bounds. + } + } + + fn manager_with_recording_store( + query_tx: UnboundedSender>, + ) -> (NetSyncManager, mpsc::Receiver) { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle().unwrap().enable("test"); + let (tx, rx) = mpsc::channel::(100); + let (_evt_tx, evt_rx) = broadcast::channel::(100); + let evt_rx = Arc::new(evt_rx); + let eventstore = RecordingEventStore { queries: query_tx } + .start() + .recipient(); + + ( + NetSyncManager::new(&bus, &tx, &evt_rx, eventstore, "my-topic"), + rx, + ) + } + + fn incoming_sync_request( + peer: PeerId, + id: u64, + limit: usize, + tx: &mpsc::Sender, + ) -> IncomingRequest { + let request: Vec = FetchEventsSince::new(AggregateId::new(1), 0, limit) + .try_into() + .unwrap(); + let responder = DirectResponder::new(id, ChannelType::Test(format!("request-{id}")), tx) + .with_request(request); + IncomingRequest { peer, responder } + } + + fn protocol_response(command: NetCommand) -> ProtocolResponse { + let NetCommand::IncomingResponse(incoming) = command else { + panic!("expected IncomingResponse, got {command:?}"); + }; + incoming.responder.to_response().unwrap().1 + } + fn local_forwardable_event(e3: &str) -> InterfoldEvent { InterfoldEvent::::new_with_timestamp( PlaintextAggregated { @@ -710,4 +883,131 @@ mod tests { "non-forwardable event should not be re-broadcast" ); } + + #[actix::test] + async fn malicious_huge_limit_is_capped_before_storage_query() { + let (query_tx, mut query_rx) = mpsc::unbounded_channel(); + let (manager, _net_rx) = manager_with_recording_store(query_tx); + let net_tx = manager.tx.clone(); + let manager = manager.start(); + + manager + .send(incoming_sync_request( + PeerId::random(), + 1, + usize::MAX, + &net_tx, + )) + .await + .unwrap(); + + let queried_limit = tokio::time::timeout(Duration::from_secs(1), query_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(queried_limit, Some(sync_scan_limit(usize::MAX) as u64)); + } + + #[actix::test] + async fn concurrent_sync_requests_are_globally_bounded() { + let (query_tx, mut query_rx) = mpsc::unbounded_channel(); + let (manager, mut net_rx) = manager_with_recording_store(query_tx); + let net_tx = manager.tx.clone(); + let manager = manager.start(); + + for id in 0..MAX_IN_FLIGHT_SYNC_REQUESTS as u64 { + manager + .send(incoming_sync_request(PeerId::random(), id, 1, &net_tx)) + .await + .unwrap(); + } + manager + .send(incoming_sync_request( + PeerId::random(), + MAX_IN_FLIGHT_SYNC_REQUESTS as u64, + 1, + &net_tx, + )) + .await + .unwrap(); + + for _ in 0..MAX_IN_FLIGHT_SYNC_REQUESTS { + tokio::time::timeout(Duration::from_secs(1), query_rx.recv()) + .await + .unwrap() + .unwrap(); + } + assert!( + query_rx.try_recv().is_err(), + "overflow request reached storage" + ); + let response = tokio::time::timeout(Duration::from_secs(1), net_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!( + protocol_response(response), + ProtocolResponse::BadRequest(reason) if reason.contains("too many in-flight") + )); + } + + #[actix::test] + async fn concurrent_sync_requests_are_bounded_per_authenticated_peer() { + let (query_tx, mut query_rx) = mpsc::unbounded_channel(); + let (manager, mut net_rx) = manager_with_recording_store(query_tx); + let net_tx = manager.tx.clone(); + let manager = manager.start(); + let peer = PeerId::random(); + + for id in 0..=MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER as u64 { + manager + .send(incoming_sync_request(peer, id, 1, &net_tx)) + .await + .unwrap(); + } + + for _ in 0..MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER { + tokio::time::timeout(Duration::from_secs(1), query_rx.recv()) + .await + .unwrap() + .unwrap(); + } + assert!( + query_rx.try_recv().is_err(), + "per-peer overflow reached storage" + ); + let response = tokio::time::timeout(Duration::from_secs(1), net_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!( + protocol_response(response), + ProtocolResponse::BadRequest(reason) if reason.contains("this peer") + )); + } + + #[actix::test] + async fn timed_out_sync_request_releases_its_in_flight_slot() { + let (query_tx, _query_rx) = mpsc::unbounded_channel(); + let (mut manager, mut net_rx) = manager_with_recording_store(query_tx); + let net_tx = manager.tx.clone(); + let peer = PeerId::random(); + let IncomingRequest { responder, .. } = incoming_sync_request(peer, 1, 1, &net_tx); + let id = CorrelationId::new(); + manager + .requests + .insert(id, PendingSyncRequest { peer, responder }); + + manager.expire_sync_request(id); + + assert!(manager.requests.is_empty()); + let response = tokio::time::timeout(Duration::from_secs(1), net_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!( + protocol_response(response), + ProtocolResponse::Error(reason) if reason.contains("timed out") + )); + } } diff --git a/crates/net/src/domain/net_event_batch.rs b/crates/net/src/domain/net_event_batch.rs index 66da028377..daa70a8d70 100644 --- a/crates/net/src/domain/net_event_batch.rs +++ b/crates/net/src/domain/net_event_batch.rs @@ -6,15 +6,111 @@ use std::fmt::Debug; -use anyhow::{Context, Result}; +use std::time::Duration; + +use anyhow::{bail, ensure, Context, Result}; use e3_events::AggregateId; +use serde::Serialize; +use tokio::time::Instant; use tracing::info; use crate::{ direct_requester::{DirectRequester, WithPeer, WithoutPeer}, + domain::sync_coordinator::effective_sync_limit, events::PeerTarget, }; +/// Startup sync is a recovery aid, not an unbounded bulk-transfer protocol. These limits prevent +/// a peer from keeping a node in startup indefinitely or retaining an attacker-controlled history +/// in memory. Operators with a larger legitimate gap must use the controlled resync procedure. +const MAX_SYNC_FETCH_PAGES: usize = 512; +const MAX_SYNC_FETCH_EVENTS: usize = 50_000; +const MAX_SYNC_FETCH_BYTES: usize = 128 * 1024 * 1024; +const MAX_SYNC_FETCH_DURATION: Duration = Duration::from_secs(5 * 60); + +pub(crate) struct SyncFetchBudget { + started: Instant, + pages: usize, + events: usize, + bytes: usize, + max_pages: usize, + max_events: usize, + max_bytes: usize, + max_duration: Duration, + exhausted: bool, +} + +impl SyncFetchBudget { + pub(crate) fn production() -> Self { + Self::new( + MAX_SYNC_FETCH_PAGES, + MAX_SYNC_FETCH_EVENTS, + MAX_SYNC_FETCH_BYTES, + ) + } + + fn new(max_pages: usize, max_events: usize, max_bytes: usize) -> Self { + Self { + started: Instant::now(), + pages: 0, + events: 0, + bytes: 0, + max_pages, + max_events, + max_bytes, + max_duration: MAX_SYNC_FETCH_DURATION, + exhausted: false, + } + } + + pub(crate) fn is_exhausted(&self) -> bool { + self.exhausted + } + + fn remaining(&mut self) -> Result { + match self.max_duration.checked_sub(self.started.elapsed()) { + Some(remaining) => Ok(remaining), + None => { + self.exhausted = true; + bail!("historical sync exceeded total deadline") + } + } + } + + fn record_page(&mut self, event_count: usize, encoded_bytes: usize) -> Result<()> { + let pages = self + .pages + .checked_add(1) + .context("sync page count overflow")?; + let events = self + .events + .checked_add(event_count) + .context("sync event count overflow")?; + let bytes = self + .bytes + .checked_add(encoded_bytes) + .context("sync byte count overflow")?; + + if pages > self.max_pages { + self.exhausted = true; + bail!("historical sync exceeded page limit ({})", self.max_pages); + } + if events > self.max_events { + self.exhausted = true; + bail!("historical sync exceeded event limit ({})", self.max_events); + } + if bytes > self.max_bytes { + self.exhausted = true; + bail!("historical sync exceeded byte limit ({})", self.max_bytes); + } + + self.pages = pages; + self.events = events; + self.bytes = bytes; + Ok(()) + } +} + #[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] pub enum BatchCursor { Done, @@ -106,7 +202,8 @@ where requester.request(request).await } -pub async fn fetch_all_batched_events( +#[cfg(test)] +async fn fetch_all_batched_events( requester: DirectRequester, peer: PeerTarget, aggregate_id: AggregateId, @@ -114,9 +211,35 @@ pub async fn fetch_all_batched_events( batch_size: usize, ) -> Result> where - E: Debug + TryFrom> + Send + Sync + 'static, + E: Debug + Serialize + TryFrom> + Send + Sync + 'static, EventBatch: TryFrom>, { + let mut budget = SyncFetchBudget::production(); + fetch_all_batched_events_with_budget( + requester, + peer, + aggregate_id, + since, + batch_size, + &mut budget, + ) + .await +} + +pub(crate) async fn fetch_all_batched_events_with_budget( + requester: DirectRequester, + peer: PeerTarget, + aggregate_id: AggregateId, + since: u128, + batch_size: usize, + budget: &mut SyncFetchBudget, +) -> Result> +where + E: Debug + Serialize + TryFrom> + Send + Sync + 'static, + EventBatch: TryFrom>, +{ + ensure!(batch_size > 0, "sync batch size must be greater than 0"); + let batch_size = effective_sync_limit(batch_size); let requester = requester.to(peer); let mut all_events = Vec::new(); let mut cursor = since; @@ -127,7 +250,27 @@ where "Fetching batch aggregate={} cursor={} batch_size={}", aggregate_id, cursor, batch_size ); - let batch = fetch_events_since(&requester, request).await?; + let remaining = budget.remaining()?; + let batch = + match tokio::time::timeout(remaining, fetch_events_since(&requester, request)).await { + Ok(result) => result?, + Err(_) => { + budget.exhausted = true; + bail!("historical sync exceeded total deadline"); + } + }; + ensure!( + batch.aggregate_id == aggregate_id, + "sync peer returned aggregate {} while fetching {}", + batch.aggregate_id, + aggregate_id + ); + ensure!( + batch.events.len() <= batch_size, + "sync peer returned {} events, exceeding requested batch size {}", + batch.events.len(), + batch_size + ); info!( "Batch received with {} events for aggregate={} cursor={}", batch.events.len(), @@ -135,11 +278,27 @@ where cursor ); + let page_bytes: usize = bincode::serialized_size(&batch.events) + .context("failed to measure historical sync page")? + .try_into() + .context("historical sync page size does not fit usize")?; + budget.record_page(batch.events.len(), page_bytes)?; + all_events + .try_reserve(batch.events.len()) + .map_err(|error| { + anyhow::anyhow!("failed to reserve historical sync buffer: {error}") + })?; all_events.extend(batch.events); match batch.next { BatchCursor::Done => break, - BatchCursor::Next(next_cursor) => cursor = next_cursor, + BatchCursor::Next(next_cursor) => { + ensure!( + next_cursor > cursor, + "sync peer returned non-advancing cursor {next_cursor} from {cursor}" + ); + cursor = next_cursor; + } } } @@ -156,6 +315,32 @@ mod tests { use std::sync::Arc; use tokio::sync::{broadcast, mpsc}; + #[test] + fn sync_fetch_budget_rejects_each_resource_limit_without_mutating_state() { + let mut budget = SyncFetchBudget::new(1, 2, 3); + budget.record_page(2, 3).unwrap(); + + let page_error = budget.record_page(0, 0).unwrap_err(); + assert!(page_error.to_string().contains("page limit")); + assert_eq!((budget.pages, budget.events, budget.bytes), (1, 2, 3)); + + let mut event_budget = SyncFetchBudget::new(2, 1, 3); + let event_error = event_budget.record_page(2, 0).unwrap_err(); + assert!(event_error.to_string().contains("event limit")); + assert_eq!( + (event_budget.pages, event_budget.events, event_budget.bytes), + (0, 0, 0) + ); + + let mut byte_budget = SyncFetchBudget::new(2, 2, 1); + let byte_error = byte_budget.record_page(0, 2).unwrap_err(); + assert!(byte_error.to_string().contains("byte limit")); + assert_eq!( + (byte_budget.pages, byte_budget.events, byte_budget.bytes), + (0, 0, 0) + ); + } + #[tokio::test] async fn test_fetch_all_batched_events() { let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); @@ -284,6 +469,37 @@ mod tests { assert_eq!(events[1], b"event2"); } + #[tokio::test] + async fn test_non_advancing_cursor_is_rejected() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + let batch = EventBatch { + events: vec![b"event1".to_vec()], + next: BatchCursor::Next(0), + aggregate_id: AggregateId::new(1), + }; + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 1)) + .respond_with(batch) + .spawn(); + + let error = fetch_all_batched_events::>( + requester, + PeerTarget::Random, + AggregateId::new(1), + 0, + 1, + ) + .await + .unwrap_err(); + + handle.await.unwrap(); + assert!(error.to_string().contains("non-advancing cursor")); + } + #[tokio::test] async fn test_single_batch_with_limit_exactly_matched_returns_done() { let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); diff --git a/crates/net/src/domain/sync_coordinator.rs b/crates/net/src/domain/sync_coordinator.rs index 77782a9c17..608bdba8c4 100644 --- a/crates/net/src/domain/sync_coordinator.rs +++ b/crates/net/src/domain/sync_coordinator.rs @@ -11,6 +11,23 @@ use crate::domain::{ net_event_batch::{BatchCursor, EventBatch, FetchEventsSince}, }; +/// Maximum number of forwardable events a remote peer may request in one sync response. +pub(crate) const MAX_SYNC_BATCH_SIZE: usize = 100; + +/// A sync page may contain non-forwardable local events which are filtered after storage. Scan a +/// small, bounded multiple of the response size so those events cannot prematurely terminate sync, +/// while preventing a request from materializing the complete remaining history. +const SYNC_SCAN_MULTIPLIER: usize = 4; +pub(crate) const MAX_SYNC_SCAN_EVENTS: usize = MAX_SYNC_BATCH_SIZE * SYNC_SCAN_MULTIPLIER; + +pub(crate) fn effective_sync_limit(requested: usize) -> usize { + requested.min(MAX_SYNC_BATCH_SIZE) +} + +pub(crate) fn sync_scan_limit(requested: usize) -> usize { + (effective_sync_limit(requested) * SYNC_SCAN_MULTIPLIER).min(MAX_SYNC_SCAN_EVENTS) +} + /// What the owning actor should do after a readiness signal. #[derive(Debug, PartialEq, Eq)] pub enum ReadinessDecision { @@ -94,38 +111,53 @@ pub enum SyncBatchOutcome { /// Build a sync response batch from the events returned by the event store. /// /// Only includes events that are safe to forward over the network: events received via gossip -/// (`Net`) and locally-produced events that are themselves gossip-forwardable. The cursor advances -/// to the timestamp of the last returned event when the limit is reached. +/// (`Net`) and locally-produced events that are themselves gossip-forwardable. The cursor is an +/// inclusive storage cursor, so it advances to one timestamp after the last returned or scanned +/// event. Both response work and storage scanning are capped independently of the peer's input. pub fn build_sync_batch( all_events: Vec, fetch: &FetchEventsSince, ) -> SyncBatchOutcome { - let limit = fetch.limit(); - if limit == 0 { + if fetch.limit() == 0 { return SyncBatchOutcome::BadRequest("limit must be greater than 0".to_string()); } + let limit = effective_sync_limit(fetch.limit()); + let scan_limit = sync_scan_limit(fetch.limit()); let aggregate_id = fetch.aggregate_id(); // Include Net events (received via gossip) and Local events that are gossip-forwardable. // Without the Local check, a node's own gossip events would be excluded from sync responses, // causing syncing peers to miss them. - let events: Vec> = all_events - .into_iter() - .filter(|e| { - e.source() == EventSource::Net - || (e.source() == EventSource::Local - && EventTranslationService::is_forwardable_event(e)) - }) - .take(limit) - .map(|ev| ev.clone_unsequenced()) - .collect(); - - let next = if events.len() == limit { - let last_event_ts = events.last().map(|e| e.ts()).unwrap_or(0); - BatchCursor::Next(last_event_ts) + let scan_was_full = all_events.len() >= scan_limit; + let mut events = Vec::with_capacity(limit); + let mut last_scanned_ts = None; + for event in all_events.into_iter().take(scan_limit) { + last_scanned_ts = Some(event.ts()); + let is_forwardable = event.source() == EventSource::Net + || (event.source() == EventSource::Local + && EventTranslationService::is_forwardable_event(&event)); + if is_forwardable { + events.push(event.clone_unsequenced()); + if events.len() == limit { + break; + } + } + } + + // Timestamp queries are inclusive. Advancing to exactly the last timestamp would repeat that + // event and, for limit=1, loop forever. When a bounded scan contains only filtered events, + // advance past the last scanned event so a later forwardable event remains reachable. + let cursor_base = if events.len() == limit { + events.last().map(|event| event.ts()) + } else if scan_was_full { + last_scanned_ts } else { - BatchCursor::Done + None }; + let next = cursor_base + .and_then(|timestamp| timestamp.checked_add(1)) + .map(BatchCursor::Next) + .unwrap_or(BatchCursor::Done); SyncBatchOutcome::Batch(EventBatch { events, @@ -226,13 +258,60 @@ mod tests { } #[test] - fn build_sync_batch_advances_cursor_when_limit_reached() { + fn build_sync_batch_limit_one_advances_past_inclusive_cursor() { let fetch = FetchEventsSince::new(AggregateId::new(1), 0, 1); let outcome = build_sync_batch(vec![net_event(5), net_event(9)], &fetch); let SyncBatchOutcome::Batch(batch) = outcome else { panic!("expected batch"); }; assert_eq!(batch.events.len(), 1); - assert!(matches!(batch.next, BatchCursor::Next(5))); + assert!(matches!(batch.next, BatchCursor::Next(6))); + } + + #[test] + fn build_sync_batch_caps_malicious_huge_limit() { + let fetch = FetchEventsSince::new(AggregateId::new(1), 0, usize::MAX); + let events = (1..=MAX_SYNC_BATCH_SIZE + 1) + .map(|ts| net_event(ts as u128)) + .collect(); + let SyncBatchOutcome::Batch(batch) = build_sync_batch(events, &fetch) else { + panic!("expected batch"); + }; + + assert_eq!(batch.events.len(), MAX_SYNC_BATCH_SIZE); + assert!(matches!( + batch.next, + BatchCursor::Next(next) if next == MAX_SYNC_BATCH_SIZE as u128 + 1 + )); + assert_eq!(sync_scan_limit(fetch.limit()), MAX_SYNC_SCAN_EVENTS); + } + + #[test] + fn build_sync_batch_advances_past_full_filtered_scan() { + let fetch = FetchEventsSince::new(AggregateId::new(1), 0, 1); + let events = (1..=sync_scan_limit(fetch.limit())) + .map(|ts| local_event(ts as u128)) + .collect(); + let SyncBatchOutcome::Batch(batch) = build_sync_batch(events, &fetch) else { + panic!("expected batch"); + }; + + assert!(batch.events.is_empty()); + assert!(matches!( + batch.next, + BatchCursor::Next(next) if next == sync_scan_limit(fetch.limit()) as u128 + 1 + )); + } + + #[test] + fn build_sync_batch_stops_at_max_timestamp() { + let fetch = FetchEventsSince::new(AggregateId::new(1), u128::MAX, 1); + let SyncBatchOutcome::Batch(batch) = build_sync_batch(vec![net_event(u128::MAX)], &fetch) + else { + panic!("expected batch"); + }; + + assert_eq!(batch.events.len(), 1); + assert!(matches!(batch.next, BatchCursor::Done)); } } diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index 008f1d643c..5164f97cef 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -88,6 +88,9 @@ pub type ProtocolResponseChannel = ResponseChannel; #[rtype("()")] /// Remote has sent us a request pub struct IncomingRequest { + /// Authenticated libp2p peer which opened the request stream. This is transport-local metadata + /// and is not part of the sync wire payload. + pub peer: PeerId, pub responder: DirectResponder, } diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index fb284ffadc..1f64fe0a0b 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -64,6 +64,24 @@ const MAX_CONSECUTIVE_DIAL_FAILURES: u32 = 40; const EVENT_CHANNEL_SIZE: usize = 1000; const CMD_CHANNEL_SIZE: usize = 1000; +/// Independent failure counters used to recover peer connectivity. +/// +/// Identity mismatches are tracked separately because ordinary dial failures +/// must not consume the one-time recovery action for a peer whose key changed. +struct PeerConnectionFailures { + dial: PeerFailureTracker, + identity_mismatch: PeerFailureTracker, +} + +impl PeerConnectionFailures { + fn new() -> Self { + Self { + dial: PeerFailureTracker::new(), + identity_mismatch: PeerFailureTracker::new(), + } + } +} + /// Returns true if the multiaddr contains a loopback IP (127.0.0.0/8 or ::1). /// Loopback addresses are only meaningful on the local machine and must not be /// added to the Kademlia routing table, otherwise they get propagated to remote @@ -185,12 +203,7 @@ impl Libp2pNetInterface { let cmd_tx = self.cmd_tx.clone(); let cmd_rx = &mut self.cmd_rx; let mut correlator = Correlator::new(); - let mut peer_failures = PeerFailureTracker::new(); - // Tracks WrongPeerId occurrences per stale peer ID, separately from - // generic dial failures: a node that was offline accumulates ordinary - // failures before coming back with new keys, and the first mismatch - // must still be treated as the first (triggering redial + bootstrap). - let mut peer_id_mismatches = PeerFailureTracker::new(); + let mut peer_failures = PeerConnectionFailures::new(); // This is to make sure we dont spam warnings in the logs let mut last_backpressure_warn = Instant::now(); @@ -247,7 +260,7 @@ impl Libp2pNetInterface { } // Process events event = self.swarm.select_next_some() => { - match process_swarm_event(&mut self.swarm, &event_tx, &cmd_tx, &mut correlator, &mut peer_failures, &mut peer_id_mismatches, &self.status, event).await { + match process_swarm_event(&mut self.swarm, &event_tx, &cmd_tx, &mut correlator, &mut peer_failures, &self.status, event).await { Ok(_) => (), Err(e) => error!("Error processing NetEvent: {e}") } @@ -329,8 +342,7 @@ async fn process_swarm_event( event_tx: &broadcast::Sender, cmd_tx: &mpsc::Sender, correlator: &mut Correlator, - peer_failures: &mut PeerFailureTracker, - peer_id_mismatches: &mut PeerFailureTracker, + peer_failures: &mut PeerConnectionFailures, status: &NetworkStatus, event: SwarmEvent, ) -> Result<()> { @@ -343,8 +355,8 @@ async fn process_swarm_event( .. } => { // Reset failure counts on successful connection - peer_failures.reset(&peer_id); - peer_id_mismatches.reset(&peer_id); + peer_failures.dial.reset(&peer_id); + peer_failures.identity_mismatch.reset(&peer_id); if num_established.get() == 1 { let total = swarm.connected_peers().count(); info!("Peer connected: {peer_id} (total: {total})"); @@ -399,10 +411,11 @@ async fn process_swarm_event( // so repeats are expected: handle them quietly (debug, no bootstrap) // to avoid flooding the logs and re-fueling the dial loop. let remote_addr = endpoint.get_remote_address().clone(); - let mismatch_count = peer_id_mismatches.record_failure(failed_peer); + let mismatch_count = + peer_failures.identity_mismatch.record_failure(failed_peer); // The stale ID is being removed from the routing table, so its // generic dial-failure history is no longer meaningful. - peer_failures.reset(failed_peer); + peer_failures.dial.reset(failed_peer); if mismatch_count == 1 { info!( "Peer ID mismatch at {remote_addr}: expected {failed_peer}, got {obtained} — \ @@ -457,14 +470,14 @@ async fn process_swarm_event( } } } else { - let count = peer_failures.record_failure(failed_peer); + let count = peer_failures.dial.record_failure(failed_peer); if count >= MAX_CONSECUTIVE_DIAL_FAILURES { info!( "Evicting unreachable peer {failed_peer} after {count} consecutive failures" ); swarm.behaviour_mut().kademlia.remove_peer(failed_peer); - peer_failures.reset(failed_peer); + peer_failures.dial.reset(failed_peer); } else { debug!( "Dial failure for {failed_peer} (attempt {count}/{MAX_CONSECUTIVE_DIAL_FAILURES}): {error}" @@ -593,21 +606,27 @@ async fn process_swarm_event( SwarmEvent::Behaviour(NodeBehaviourEvent::RequestResponse( RequestResponseEvent::Message { + peer, message: RequestResponseMessage::Request { request, channel, request_id, }, - .. }, )) => { - debug!("Incoming request received (id={})", request_id); + debug!( + "Incoming request received (peer={}, id={})", + peer, request_id + ); let responder = DirectResponder::new(request_id, ChannelType::Channel(channel), cmd_tx) .with_request(request); // received a request for events - event_tx.send(NetEvent::IncomingRequest(IncomingRequest { responder }))?; + event_tx.send(NetEvent::IncomingRequest(IncomingRequest { + peer, + responder, + }))?; } SwarmEvent::Behaviour(NodeBehaviourEvent::RequestResponse( From ea5b2259b5436efd8bd0214572d084bce49086d2 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:37:49 +0500 Subject: [PATCH 11/62] fix(dappnode): fail closed on credentials and health --- dappnode/Dockerfile | 11 +- dappnode/README.md | 60 +++-- dappnode/config.template.yaml | 9 +- dappnode/docker-compose.yml | 6 +- dappnode/entrypoint.sh | 161 ++++++++++---- dappnode/healthcheck.sh | 40 ++++ dappnode/setup-wizard.yml | 51 ++--- dappnode/tests/test-hardening.sh | 230 ++++++++++++++++++++ deploy/swarm_deployment.md | 10 +- docs/pages/ciphernode-operators/running.mdx | 65 +++++- 10 files changed, 531 insertions(+), 112 deletions(-) create mode 100644 dappnode/healthcheck.sh create mode 100644 dappnode/tests/test-hardening.sh diff --git a/dappnode/Dockerfile b/dappnode/Dockerfile index 95fa27e2a3..63ed2adfbd 100644 --- a/dappnode/Dockerfile +++ b/dappnode/Dockerfile @@ -10,16 +10,18 @@ FROM debian:stable-slim # Install runtime dependencies RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ - procps \ gettext-base \ + iproute2 \ + jq \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* # Create non-root user RUN useradd -m -u 1000 -s /bin/bash ciphernode -# Create data directory -RUN mkdir -p /data && chown ciphernode:ciphernode /data +# Create writable data and ephemeral credential-upload directories. +RUN install -d -o ciphernode -g ciphernode -m 0750 /data \ + && install -d -o ciphernode -g ciphernode -m 0700 /run/secrets # Copy binary from upstream COPY --from=upstream /usr/local/bin/interfold /usr/local/bin/interfold @@ -27,6 +29,7 @@ RUN chmod +x /usr/local/bin/interfold # Copy entrypoint and config template COPY --chmod=755 entrypoint.sh /usr/local/bin/entrypoint.sh +COPY --chmod=755 healthcheck.sh /usr/local/bin/healthcheck.sh COPY --chmod=644 config.template.yaml /opt/config.template.yaml # Labels @@ -38,6 +41,6 @@ USER ciphernode WORKDIR /data HEALTHCHECK --interval=30s --timeout=10s --start-period=60s --retries=3 \ - CMD pgrep -f interfold || exit 1 + CMD ["/usr/local/bin/healthcheck.sh"] ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/dappnode/README.md b/dappnode/README.md index 63ef510562..0b9c2ac747 100644 --- a/dappnode/README.md +++ b/dappnode/README.md @@ -27,15 +27,17 @@ dappnode/ ├── Dockerfile # Builds the DAppNode image from the upstream ciphernode image ├── docker-compose.yml # DAppNode service definition (single variant) ├── dappnode_package.json # Package metadata (name, version, links, backup, etc.) -├── setup-wizard.yml # DAppNode UI form -> environment variables +├── setup-wizard.yml # DAppNode UI form -> configuration and credential upload ├── entrypoint.sh # Startup script (validates env, renders config, runs interfold) +├── healthcheck.sh # Local process, credential, config, and QUIC listener checks ├── config.template.yaml # Interfold config template (filled via envsubst) ├── releases.json # Release metadata used by DAppNode └── avatar-default.png # Icon shown in the DAppNode UI ``` -All configuration is done via **environment variables**, wired through `docker-compose.yml` and -`setup-wizard.yml`. +Non-secret configuration is supplied through environment variables. Credentials use DAppNode's +`fileUpload` setup-wizard target and are copied to `/run/secrets/secrets.json` before startup; they +are never accepted through container environment variables. ## Quick Start @@ -49,7 +51,8 @@ Once this package is published to the DAppStore: - `RPC_URL` – WebSocket RPC endpoint (e.g. `wss://ethereum-sepolia-rpc.publicnode.com`) - `NETWORK` – e.g. `sepolia`, `mainnet`, `localhost` - Contract addresses + deploy blocks - - Optional keys and peers + - A required ciphernode credentials JSON file + - Optional peers 4. Confirm and finish the installation. 5. Go to **Packages → interfold-ciphernode.public.dappnode.eth → Logs** to verify the node started @@ -115,7 +118,7 @@ Fill in the wizard fields, then install. ## Configuration -All runtime configuration is done via environment variables. They are: +Non-secret runtime configuration is provided through environment variables: ### Core @@ -150,16 +153,29 @@ Used to populate the `chains[0].contracts` section in `config.yaml`: These are all required in the setup wizard so that the node can index chain events from the correct block heights. -### Secrets and keys +### Credentials file -- **`ENCRYPTION_PASSWORD`** Optional local encryption password. If set, `entrypoint.sh` calls: - - `interfold password set --config /data/config.yaml` +Create a local JSON file containing all three credential values: -- **`NETWORK_PRIVATE_KEY`** Optional libp2p network key. If set, `entrypoint.sh` calls: - - `interfold net set-key --config /data/config.yaml --net-keypair "$NETWORK_PRIVATE_KEY"` +```json +{ + "password": "a strong encryption password", + "private_key": "0x<64 hex characters>", + "network_private_key": "0x<64 hex characters>" +} +``` + +Upload it in the setup wizard as **Ciphernode Credentials JSON**. DAppNode copies it to +`/run/secrets/secrets.json` before starting the container. The entrypoint validates a maximum size +of 16 KiB, required fields, and key encodings, then runs the exact commands supported by the pinned +Interfold v0.1.8 image. Any failed command aborts startup. Wallet and network keys are encrypted in +`/data`; v0.1.8 stores its password key there as a mode-`0400` file. After successful persistence, +the entrypoint removes the combined plaintext upload. -- **`PRIVATE_KEY`** Optional Ethereum private key (hex). If set, `entrypoint.sh` calls: - - `interfold wallet set --config /data/config.yaml --private-key "$PRIVATE_KEY"` +The Ethereum key must correspond to `NODE_ADDRESS`. Keep an encrypted offline backup of this JSON; +do not paste its contents into package environment variables, `EXTRA_OPTS`, logs, or support +bundles. On an ordinary restart, persisted credential state is reused from `/data` and the upload is +not required again. Uploading a different password while state already exists fails closed. ### Peers @@ -190,7 +206,8 @@ At container startup, `entrypoint.sh`: - network name and RPC URL - contract addresses and deploy blocks -4. Optionally programs password, network key, and wallet key via the `interfold` CLI. +4. Validates the uploaded credential file and programs password, network key, and wallet key. The + pinned v0.1.8 network-key command is `interfold net keypair set`; failures stop startup. 5. Builds CLI args, including verbosity and `--peer` flags from `PEERS`. 6. Executes: @@ -199,7 +216,22 @@ At container startup, `entrypoint.sh`: ``` The state and databases live under `/data` inside the container, which is backed by the -`ciphernode_data` Docker volume and listed as a backup target in `dappnode_package.json`. +`ciphernode_data` Docker volume and listed as a backup target in `dappnode_package.json`. Because +that volume includes the password key, DAppNode backups of `/data` must be encrypted and +access-controlled even though the wallet and network keys inside the volume are encrypted. + +## Health semantics + +Interfold v0.1.8 does not expose a local readiness endpoint. The package health check therefore uses +the strongest non-invasive local signals available in that release: PID 1 must be the expected +`interfold start` command using `/data/config.yaml`, the protected config/password files must exist, +the v0.1.8 Sled/event-log directories must be initialized, and the configured QUIC UDP listener must +be bound. This detects the old false-positive case where an unrelated process matched `pgrep`, as +well as missing credentials, uninitialized persistence, and a dead network listener. + +This remains a liveness/startup check, not proof of canonical chain sync, healthy RPC responses, +honest peers, registration, or safe protocol participation. Operators must inspect logs and on-chain +status before treating the node as protocol-ready. ## Data & Ports diff --git a/dappnode/config.template.yaml b/dappnode/config.template.yaml index a90484afea..8d8be36550 100644 --- a/dappnode/config.template.yaml +++ b/dappnode/config.template.yaml @@ -4,9 +4,12 @@ node: address: '${NODE_ADDRESS}' quic_port: ${QUIC_PORT} - autonetkey: true - autopassword: true - autowallet: true + # Credentials are provisioned from /run/secrets/secrets.json before start. + # Fail closed if required credential state is absent instead of generating a + # different identity during a restart or maintenance CLI invocation. + autonetkey: false + autopassword: false + autowallet: false chains: - name: '${NETWORK}' diff --git a/dappnode/docker-compose.yml b/dappnode/docker-compose.yml index b23ea48a6d..63902add83 100644 --- a/dappnode/docker-compose.yml +++ b/dappnode/docker-compose.yml @@ -21,10 +21,6 @@ services: PEERS: '' LOG_LEVEL: 'info' EXTRA_OPTS: '' - # Secrets - ENCRYPTION_PASSWORD: '' - NETWORK_PRIVATE_KEY: '' - PRIVATE_KEY: '' # Contracts (variants override these) INTERFOLD_CONTRACT: '' CIPHERNODE_REGISTRY_CONTRACT: '' @@ -35,7 +31,7 @@ services: security_opt: - no-new-privileges:true healthcheck: - test: ['CMD', 'pgrep', '-f', 'interfold'] + test: ['CMD', '/usr/local/bin/healthcheck.sh'] interval: 30s timeout: 10s retries: 3 diff --git a/dappnode/entrypoint.sh b/dappnode/entrypoint.sh index e7024b0cb5..192a43c0ed 100644 --- a/dappnode/entrypoint.sh +++ b/dappnode/entrypoint.sh @@ -1,78 +1,153 @@ #!/bin/bash # DAppNode Interfold Ciphernode Entrypoint -set -e +set -Eeuo pipefail -CONFIG_DIR="/data" -CONFIG_FILE="$CONFIG_DIR/config.yaml" -TEMPLATE_FILE="/opt/config.template.yaml" +umask 077 -log() { echo "[$(date '+%H:%M:%S')] $1"; } +CONFIG_DIR="${CONFIG_DIR:-/data}" +CONFIG_FILE="${CONFIG_FILE:-$CONFIG_DIR/config.yaml}" +TEMPLATE_FILE="${TEMPLATE_FILE:-/opt/config.template.yaml}" +SECRETS_FILE="${SECRETS_FILE:-/run/secrets/secrets.json}" +# Interfold v0.1.8 resolves a relative `key_file: key` beside a discovered +# /data/config.yaml to this path for the default node profile. +PASSWORD_FILE="${PASSWORD_FILE:-$CONFIG_DIR/.enclave/config/_default/key}" + +log() { printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$1"; } +fail() { + log "ERROR: $1" + exit 1 +} echo "==========================================" echo " Interfold Ciphernode - ${NETWORK:-sepolia}" echo "==========================================" -# Validate RPC URL (required) -if [ -z "$RPC_URL" ]; then - log "ERROR: RPC_URL is required!" - log "Set it in the DAppNode package configuration." - exit 1 +# Environment variables are visible in Docker/DAppNode metadata. Refuse the +# legacy secret injection contract instead of silently preferring one source. +if [ -n "${ENCRYPTION_PASSWORD:-}" ] || [ -n "${NETWORK_PRIVATE_KEY:-}" ] || [ -n "${PRIVATE_KEY:-}" ]; then + fail "credential environment variables are unsupported; upload the DAppNode credentials JSON file" fi -if [[ ! "$RPC_URL" =~ ^wss?:// ]]; then - log "ERROR: RPC_URL must be a WebSocket URL (ws:// or wss://)" - exit 1 -fi +# Validate RPC URL (required). +[ -n "${RPC_URL:-}" ] || fail "RPC_URL is required; set it in the DAppNode package configuration" +[[ "$RPC_URL" =~ ^wss?:// ]] || fail "RPC_URL must be a WebSocket URL (ws:// or wss://)" + +[ -r "$TEMPLATE_FILE" ] || fail "configuration template is not readable: $TEMPLATE_FILE" +mkdir -p "$CONFIG_DIR" -# Set defaults +# Set non-secret defaults. export NETWORK="${NETWORK:-sepolia}" export QUIC_PORT="${QUIC_PORT:-37173}" export NODE_ADDRESS="${NODE_ADDRESS:-}" export LOG_LEVEL="${LOG_LEVEL:-info}" -# Contract addresses are set by package variants or environment variables -# No need for default placeholders here as variants handle network-specific values +case "$LOG_LEVEL" in + info|debug|trace) ;; + *) fail "LOG_LEVEL must be one of: info, debug, trace" ;; +esac -# Generate config from template +# Generate config from the fixed template. The 0077 umask keeps RPC +# credentials in the rendered URL out of group/world-readable files. log "Generating configuration..." envsubst < "$TEMPLATE_FILE" > "$CONFIG_FILE" - -# Setup secrets if provided -if [ -n "$ENCRYPTION_PASSWORD" ]; then - log "Setting encryption password..." - echo "$ENCRYPTION_PASSWORD" | interfold password set --config "$CONFIG_FILE" 2>/dev/null || true -fi - -if [ -n "$NETWORK_PRIVATE_KEY" ]; then +chmod 600 "$CONFIG_FILE" + +validate_secret_file() { + [ -f "$SECRETS_FILE" ] || fail "credentials path is not a regular file: $SECRETS_FILE" + [ ! -L "$SECRETS_FILE" ] || fail "credentials path must not be a symbolic link: $SECRETS_FILE" + [ -r "$SECRETS_FILE" ] || fail "credentials file is not readable: $SECRETS_FILE" + + local size + size=$(wc -c < "$SECRETS_FILE") + [ "$size" -le 16384 ] || fail "credentials file exceeds the 16 KiB limit" + + jq -e ' + type == "object" and + (keys | sort == ["network_private_key", "password", "private_key"]) and + (.password | type == "string" and length > 0 and length <= 1024 and + test("^[^\\r\\n\\u0000]+$") and . == gsub("^\\s+|\\s+$"; "")) and + (.private_key | type == "string" and test("^0x[0-9a-fA-F]{64}$")) and + (.network_private_key | type == "string" and test("^0x[0-9a-fA-F]{64}$")) + ' "$SECRETS_FILE" >/dev/null || fail "credentials file must contain valid password, private_key, and network_private_key strings" +} + +validate_persisted_password_file() { + [ -f "$PASSWORD_FILE" ] || fail "persisted password path is not a regular file: $PASSWORD_FILE" + [ ! -L "$PASSWORD_FILE" ] || fail "persisted password path must not be a symbolic link: $PASSWORD_FILE" + chmod 400 "$PASSWORD_FILE" || fail "could not restrict persisted password permissions" + [ -r "$PASSWORD_FILE" ] || fail "persisted password file is not readable: $PASSWORD_FILE" +} + +configure_credentials() { + validate_secret_file + + local password private_key network_private_key + password=$(jq -er '.password' "$SECRETS_FILE") + private_key=$(jq -er '.private_key' "$SECRETS_FILE") + network_private_key=$(jq -er '.network_private_key' "$SECRETS_FILE") + + if [ -e "$PASSWORD_FILE" ]; then + validate_persisted_password_file + printf '%s' "$password" | cmp -s - "$PASSWORD_FILE" || fail "uploaded password does not match the persisted credential key" + log "Using the matching persisted encryption password." + else + log "Setting encryption password..." + interfold password set --config "$CONFIG_FILE" --password "$password" + fi + + # v0.1.8 nests this operation under `net keypair`; `net set-key` is not a + # valid command for the image pinned by this package. log "Setting network key..." - interfold net set-key --config "$CONFIG_FILE" --net-keypair "$NETWORK_PRIVATE_KEY" 2>/dev/null || true -fi + interfold net keypair set --config "$CONFIG_FILE" --net-keypair "$network_private_key" -if [ -n "$PRIVATE_KEY" ]; then log "Setting wallet key..." - interfold wallet set --config "$CONFIG_FILE" --private-key "$PRIVATE_KEY" 2>/dev/null || true + interfold wallet set --config "$CONFIG_FILE" --private-key "$private_key" + + unset password private_key network_private_key + + # DAppNode copies fileUpload content into this container before startup. + # Wallet/network keys are encrypted in /data and v0.1.8 stores the password + # key there with mode 0400. Remove the combined plaintext upload. + rm -f "$SECRETS_FILE" + log "Credential setup completed." +} + +if [ -e "$SECRETS_FILE" ]; then + configure_credentials +elif [ -s "$PASSWORD_FILE" ]; then + # Backward-compatible restart/upgrade path: DAppNode file uploads are copied + # when configuring a container, while encrypted credentials persist in + # /data. Interfold itself will fail startup if wallet/network state is absent. + validate_persisted_password_file + log "No credential upload present; using persisted credential state." +else + fail "credentials file is required for first startup: $SECRETS_FILE" fi -# Build CLI args -CLI_ARGS="--config $CONFIG_FILE" +# Build CLI args without shell evaluation. +CLI_ARGS=(--config "$CONFIG_FILE") case "$LOG_LEVEL" in - trace) CLI_ARGS="-vvv $CLI_ARGS" ;; - debug) CLI_ARGS="-vv $CLI_ARGS" ;; - info) CLI_ARGS="-v $CLI_ARGS" ;; + trace) CLI_ARGS=(-vvv "${CLI_ARGS[@]}") ;; + debug) CLI_ARGS=(-vv "${CLI_ARGS[@]}") ;; + info) CLI_ARGS=(-v "${CLI_ARGS[@]}") ;; esac -# Add peers if provided -if [ -n "$PEERS" ]; then +# Add peers if provided. +if [ -n "${PEERS:-}" ]; then IFS=',' read -ra PEER_ARRAY <<< "$PEERS" for peer in "${PEER_ARRAY[@]}"; do - peer=$(echo "$peer" | xargs) - [ -n "$peer" ] && CLI_ARGS="$CLI_ARGS --peer $peer" + peer="$(echo "$peer" | xargs)" + [ -n "$peer" ] && CLI_ARGS+=(--peer "$peer") done fi -[ -n "$EXTRA_OPTS" ] && CLI_ARGS="$CLI_ARGS $EXTRA_OPTS" +# EXTRA_OPTS remains an advanced, non-secret escape hatch. Split it as plain +# arguments; never evaluate it as shell source. +if [ -n "${EXTRA_OPTS:-}" ]; then + read -r -a EXTRA_ARGS <<< "$EXTRA_OPTS" + CLI_ARGS+=("${EXTRA_ARGS[@]}") +fi -# Start -log "Starting: interfold start $CLI_ARGS" -exec interfold start $CLI_ARGS \ No newline at end of file +log "Starting Interfold ciphernode." +exec interfold start "${CLI_ARGS[@]}" diff --git a/dappnode/healthcheck.sh b/dappnode/healthcheck.sh new file mode 100644 index 0000000000..83cd82faaf --- /dev/null +++ b/dappnode/healthcheck.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# Best available local health signal for the v0.1.8 image. That release has no +# readiness HTTP/control endpoint, so require the expected PID 1, its exact +# start/config arguments, protected local credential/config files, and the QUIC +# listener. This is stronger than PID-only liveness but is not a chain-sync or +# protocol-readiness guarantee. +set -eu + +PROC_ROOT="${PROC_ROOT:-/proc}" +CONFIG_FILE="${CONFIG_FILE:-/data/config.yaml}" +PASSWORD_FILE="${PASSWORD_FILE:-/data/.enclave/config/_default/key}" +DB_PATH="${DB_PATH:-/data/.enclave/data/_default/db}" +EVENT_LOG_PATH="${EVENT_LOG_PATH:-/data/.enclave/data/_default/log.0}" +QUIC_PORT="${QUIC_PORT:-37173}" +SS_BIN="${SS_BIN:-ss}" +STAT_BIN="${STAT_BIN:-stat}" + +case "$QUIC_PORT" in + ''|*[!0-9]*) exit 1 ;; +esac +[ "$QUIC_PORT" -ge 1 ] && [ "$QUIC_PORT" -le 65535 ] || exit 1 + +[ -r "$PROC_ROOT/1/cmdline" ] || exit 1 +[ "$(basename "$(readlink "$PROC_ROOT/1/exe")")" = "interfold" ] || exit 1 + +cmdline=$(tr '\000' '\n' < "$PROC_ROOT/1/cmdline") +printf '%s\n' "$cmdline" | grep -Fxq 'start' || exit 1 +printf '%s\n' "$cmdline" | grep -Fxq "$CONFIG_FILE" || exit 1 + +[ -s "$CONFIG_FILE" ] || exit 1 +[ -s "$PASSWORD_FILE" ] || exit 1 +[ -d "$DB_PATH" ] || exit 1 +[ -d "$EVENT_LOG_PATH" ] || exit 1 + +config_mode=$($STAT_BIN -c '%a' "$CONFIG_FILE") +password_mode=$($STAT_BIN -c '%a' "$PASSWORD_FILE") +case "$config_mode" in 400|600) ;; *) exit 1 ;; esac +case "$password_mode" in 400|600) ;; *) exit 1 ;; esac + +$SS_BIN -H -u -l -n "sport = :$QUIC_PORT" | grep -q . || exit 1 diff --git a/dappnode/setup-wizard.yml b/dappnode/setup-wizard.yml index 804c0a4753..c628cd4754 100644 --- a/dappnode/setup-wizard.yml +++ b/dappnode/setup-wizard.yml @@ -37,22 +37,11 @@ fields: name: NODE_ADDRESS service: ciphernode title: 'Node Address' - description: 'Public address for this node (must match the private key below if set).' + description: 'Public address for this node (must match private_key in the credentials JSON file).' required: true pattern: '^0x[a-fA-F0-9]{40}$' patternErrorMessage: 'Must be a valid 20-byte hex address (0x...)' - - id: PRIVATE_KEY - target: - type: environment - name: PRIVATE_KEY - service: ciphernode - title: 'Private Key' - description: 'Private key for this node. Must correspond to the Node Address above.' - secret: true - required: false - pattern: '^(0x[a-fA-F0-9]{64})?$' - # Contracts - id: INTERFOLD_CONTRACT target: @@ -132,27 +121,29 @@ fields: Leave empty for auto-discovery. required: false - # Secrets - - id: ENCRYPTION_PASSWORD + # Credentials are uploaded as a file so they are not stored in container + # environment/process metadata. DAppNode copies the file before startup. + - id: CREDENTIALS_FILE target: - type: environment - name: ENCRYPTION_PASSWORD + type: fileUpload + path: /run/secrets/secrets.json service: ciphernode - title: 'Encryption Password' - description: 'Password for local encryption. Leave empty to auto-generate.' - secret: true - required: false + title: 'Ciphernode Credentials JSON' + description: | + Upload a JSON file with exactly the credentials used by this node: - - id: NETWORK_PRIVATE_KEY - target: - type: environment - name: NETWORK_PRIVATE_KEY - service: ciphernode - title: 'Network Private Key' - description: 'libp2p network key for consistent peer ID across restarts. Leave empty to auto-generate.' - secret: true - required: false - pattern: '^(0x[a-fA-F0-9]{64})?$' + ```json + { + "password": "a strong encryption password", + "private_key": "0x<64 hex characters>", + "network_private_key": "0x<64 hex characters>" + } + ``` + + The Ethereum private key must correspond to the Node Address above. Keep + an encrypted backup; the plaintext upload is removed after successful + credential provisioning. + required: true # Advanced - id: LOG_LEVEL diff --git a/dappnode/tests/test-hardening.sh b/dappnode/tests/test-hardening.sh new file mode 100644 index 0000000000..aeedb1673d --- /dev/null +++ b/dappnode/tests/test-hardening.sh @@ -0,0 +1,230 @@ +#!/bin/bash +set -Eeuo pipefail + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +TEST_ROOT=$(mktemp -d) +trap 'rm -rf "$TEST_ROOT"' EXIT + +fail() { + printf 'FAIL: %s\n' "$1" >&2 + exit 1 +} + +assert_contains() { + grep -Fq "$2" "$1" || fail "expected '$2' in $1" +} + +assert_not_contains() { + if grep -Fq "$2" "$1"; then + fail "did not expect '$2' in $1" + fi +} + +make_mock_interfold() { + local bin_dir=$1 + mkdir -p "$bin_dir" + printf '%s\n' \ + '#!/bin/bash' \ + 'set -Eeuo pipefail' \ + 'case "${1:-} ${2:-} ${3:-}" in' \ + ' "password set "*) operation=password ;;' \ + ' "net keypair set"*) operation=network ;;' \ + ' "wallet set "*) operation=wallet ;;' \ + ' "start "*) operation=start ;;' \ + ' *) operation=unexpected ;;' \ + 'esac' \ + 'printf "%s\n" "$operation" >> "$CALL_LOG"' \ + '[ "${FAIL_ON:-}" != "$operation" ] || exit 42' \ + 'if [ "$operation" = password ]; then' \ + ' password=' \ + ' while [ "$#" -gt 0 ]; do' \ + ' if [ "$1" = --password ]; then password=$2; break; fi' \ + ' shift' \ + ' done' \ + ' mkdir -p "$(dirname "$PASSWORD_FILE")"' \ + ' printf "%s" "$password" > "$PASSWORD_FILE"' \ + ' chmod 400 "$PASSWORD_FILE"' \ + 'fi' \ + '[ "$operation" != unexpected ]' \ + > "$bin_dir/interfold" + chmod +x "$bin_dir/interfold" +} + +write_secrets() { + local path=$1 + printf '%s\n' '{' \ + ' "password": "correct horse battery staple",' \ + ' "private_key": "0x1111111111111111111111111111111111111111111111111111111111111111",' \ + ' "network_private_key": "0x2222222222222222222222222222222222222222222222222222222222222222"' \ + '}' > "$path" +} + +run_entrypoint() { + local case_dir=$1 + shift + mkdir -p "$case_dir/data" "$case_dir/secrets" "$case_dir/bin" + : > "$case_dir/calls" + make_mock_interfold "$case_dir/bin" + + env -u ENCRYPTION_PASSWORD -u NETWORK_PRIVATE_KEY -u PRIVATE_KEY \ + PATH="$case_dir/bin:$PATH" \ + CONFIG_DIR="$case_dir/data" \ + CONFIG_FILE="$case_dir/data/config.yaml" \ + TEMPLATE_FILE="$ROOT_DIR/config.template.yaml" \ + SECRETS_FILE="$case_dir/secrets/secrets.json" \ + PASSWORD_FILE="$case_dir/data/password" \ + CALL_LOG="$case_dir/calls" \ + RPC_URL="ws://127.0.0.1:8545" \ + NODE_ADDRESS="0x3333333333333333333333333333333333333333" \ + INTERFOLD_CONTRACT="0x4444444444444444444444444444444444444444" \ + CIPHERNODE_REGISTRY_CONTRACT="0x5555555555555555555555555555555555555555" \ + BONDING_REGISTRY_CONTRACT="0x6666666666666666666666666666666666666666" \ + INTERFOLD_DEPLOY_BLOCK=1 \ + CIPHERNODE_REGISTRY_DEPLOY_BLOCK=2 \ + BONDING_REGISTRY_DEPLOY_BLOCK=3 \ + PRIVATE_KEY="${TEST_PRIVATE_KEY:-}" \ + "$@" bash "$ROOT_DIR/entrypoint.sh" > "$case_dir/output" 2>&1 +} + +# Successful provisioning uses the exact v0.1.8 commands, removes the +# plaintext upload, and starts only after every credential command succeeds. +success_dir="$TEST_ROOT/success" +mkdir -p "$success_dir/secrets" +write_secrets "$success_dir/secrets/secrets.json" +run_entrypoint "$success_dir" +[ ! -e "$success_dir/secrets/secrets.json" ] || fail "successful setup retained plaintext credentials" +[ "$(tr '\n' ' ' < "$success_dir/calls")" = "password network wallet start " ] || fail "unexpected successful command order" +assert_contains "$success_dir/data/config.yaml" 'autopassword: false' +assert_contains "$success_dir/data/config.yaml" 'autonetkey: false' +assert_contains "$success_dir/data/config.yaml" 'autowallet: false' + +# A credential command failure must propagate and must never start the node. +failure_dir="$TEST_ROOT/failure" +mkdir -p "$failure_dir/secrets" +write_secrets "$failure_dir/secrets/secrets.json" +if run_entrypoint "$failure_dir" FAIL_ON=network; then + fail "network credential failure was ignored" +fi +assert_contains "$failure_dir/calls" 'network' +assert_not_contains "$failure_dir/calls" 'start' +[ -e "$failure_dir/secrets/secrets.json" ] || fail "failed setup removed recovery input" + +# Existing state may only be reused with the password that encrypted it. +mismatch_dir="$TEST_ROOT/password-mismatch" +mkdir -p "$mismatch_dir/data" "$mismatch_dir/secrets" +printf '%s' 'different-password' > "$mismatch_dir/data/password" +write_secrets "$mismatch_dir/secrets/secrets.json" +if run_entrypoint "$mismatch_dir"; then + fail "mismatched persisted password was accepted" +fi +[ ! -s "$mismatch_dir/calls" ] || fail "password mismatch mutated credentials" + +# Malformed or absent first-start credentials fail before invoking Interfold. +malformed_dir="$TEST_ROOT/malformed" +mkdir -p "$malformed_dir/secrets" +printf '%s\n' '{"password":"only-one-field"}' > "$malformed_dir/secrets/secrets.json" +if run_entrypoint "$malformed_dir"; then + fail "malformed credentials were accepted" +fi +[ ! -s "$malformed_dir/calls" ] || fail "malformed credentials invoked Interfold" + +missing_dir="$TEST_ROOT/missing" +if run_entrypoint "$missing_dir"; then + fail "first startup without credentials was accepted" +fi +[ ! -s "$missing_dir/calls" ] || fail "missing credentials invoked Interfold" + +# A normal restart may reuse credentials already encrypted in the persistent +# /data volume without requiring the one-time plaintext upload again. +restart_dir="$TEST_ROOT/restart" +mkdir -p "$restart_dir/data" +printf '%s' 'persisted-password' > "$restart_dir/data/password" +chmod 400 "$restart_dir/data/password" +run_entrypoint "$restart_dir" +[ "$(tr '\n' ' ' < "$restart_dir/calls")" = "start " ] || fail "persisted restart unexpectedly re-provisioned credentials" + +# Legacy secret environment variables are explicitly rejected. +legacy_dir="$TEST_ROOT/legacy-env" +mkdir -p "$legacy_dir/data" +printf '%s' 'persisted-password' > "$legacy_dir/data/password" +if TEST_PRIVATE_KEY=legacy run_entrypoint "$legacy_dir"; then + fail "legacy secret environment variable was accepted" +fi + +# Health probe regression: require the exact process/config, protected files, +# and bound QUIC listener rather than accepting an arbitrary matching PID. +health_dir="$TEST_ROOT/health" +mkdir -p "$health_dir/proc/1" "$health_dir/bin" "$health_dir/data" "$health_dir/data/db" "$health_dir/data/log.0" +ln -s "$health_dir/bin/interfold" "$health_dir/proc/1/exe" +: > "$health_dir/bin/interfold" +printf '/usr/local/bin/interfold\0start\0-v\0--config\0%s\0' "$health_dir/data/config.yaml" > "$health_dir/proc/1/cmdline" +printf 'config' > "$health_dir/data/config.yaml" +printf 'password' > "$health_dir/data/password" + +printf '%s\n' \ + '#!/bin/sh' \ + '[ "${SS_READY:-1}" = 1 ] && printf "udp listener\n"' \ + > "$health_dir/bin/ss" +printf '%s\n' \ + '#!/bin/sh' \ + 'printf "%s\n" "${STAT_MODE:-600}"' \ + > "$health_dir/bin/stat" +chmod +x "$health_dir/bin/ss" "$health_dir/bin/stat" + +PROC_ROOT="$health_dir/proc" \ +CONFIG_FILE="$health_dir/data/config.yaml" \ +PASSWORD_FILE="$health_dir/data/password" \ +DB_PATH="$health_dir/data/db" \ +EVENT_LOG_PATH="$health_dir/data/log.0" \ +SS_BIN="$health_dir/bin/ss" \ +STAT_BIN="$health_dir/bin/stat" \ +sh "$ROOT_DIR/healthcheck.sh" || fail "healthy local state was rejected" + +if PROC_ROOT="$health_dir/proc" \ + CONFIG_FILE="$health_dir/data/config.yaml" \ + PASSWORD_FILE="$health_dir/data/password" \ + DB_PATH="$health_dir/data/db" \ + EVENT_LOG_PATH="$health_dir/data/log.0" \ + SS_BIN="$health_dir/bin/ss" \ + STAT_BIN="$health_dir/bin/stat" \ + SS_READY=0 sh "$ROOT_DIR/healthcheck.sh"; then + fail "missing QUIC listener was considered healthy" +fi + +if PROC_ROOT="$health_dir/proc" \ + CONFIG_FILE="$health_dir/data/config.yaml" \ + PASSWORD_FILE="$health_dir/data/password" \ + DB_PATH="$health_dir/data/db" \ + EVENT_LOG_PATH="$health_dir/data/log.0" \ + SS_BIN="$health_dir/bin/ss" \ + STAT_BIN="$health_dir/bin/stat" \ + STAT_MODE=644 sh "$ROOT_DIR/healthcheck.sh"; then + fail "insecure credential permissions were considered healthy" +fi + +ln -sfn "$health_dir/bin/not-interfold" "$health_dir/proc/1/exe" +if PROC_ROOT="$health_dir/proc" \ + CONFIG_FILE="$health_dir/data/config.yaml" \ + PASSWORD_FILE="$health_dir/data/password" \ + DB_PATH="$health_dir/data/db" \ + EVENT_LOG_PATH="$health_dir/data/log.0" \ + SS_BIN="$health_dir/bin/ss" \ + STAT_BIN="$health_dir/bin/stat" \ + sh "$ROOT_DIR/healthcheck.sh"; then + fail "unrelated PID 1 was considered healthy" +fi + +ln -sfn "$health_dir/bin/interfold" "$health_dir/proc/1/exe" +rmdir "$health_dir/data/log.0" +if PROC_ROOT="$health_dir/proc" \ + CONFIG_FILE="$health_dir/data/config.yaml" \ + PASSWORD_FILE="$health_dir/data/password" \ + DB_PATH="$health_dir/data/db" \ + EVENT_LOG_PATH="$health_dir/data/log.0" \ + SS_BIN="$health_dir/bin/ss" \ + STAT_BIN="$health_dir/bin/stat" \ + sh "$ROOT_DIR/healthcheck.sh"; then + fail "uninitialized event persistence was considered healthy" +fi + +printf 'PASS: DAppNode credential and health hardening regressions\n' diff --git a/deploy/swarm_deployment.md b/deploy/swarm_deployment.md index 32ca5372cc..519c031d67 100644 --- a/deploy/swarm_deployment.md +++ b/deploy/swarm_deployment.md @@ -126,7 +126,7 @@ Remember to modify any highlighted files before use with unique secrets. To deploy ``` -./deploy/deploy.sh interfold ghcr.io/gnosisguild/ciphernode:latest +./deploy/deploy.sh interfold ghcr.io/gnosisguild/ciphernode: ``` This will deploy the following services: @@ -134,10 +134,10 @@ This will deploy the following services: ``` ❯ docker service ls ID NAME MODE REPLICAS IMAGE PORTS -tr44go8vevh1 interfold_cn4 replicated 1/1 ghcr.io/gnosisguild/ciphernode:latest -kdqktv85xcuv interfold_cn1 replicated 1/1 ghcr.io/gnosisguild/ciphernode:latest -nguul381w6mu interfold_cn2 replicated 1/1 ghcr.io/gnosisguild/ciphernode:latest -zgmwmv7cd63j interfold_cn3 replicated 1/1 ghcr.io/gnosisguild/ciphernode:latest +tr44go8vevh1 interfold_cn4 replicated 1/1 ghcr.io/gnosisguild/ciphernode: +kdqktv85xcuv interfold_cn1 replicated 1/1 ghcr.io/gnosisguild/ciphernode: +nguul381w6mu interfold_cn2 replicated 1/1 ghcr.io/gnosisguild/ciphernode: +zgmwmv7cd63j interfold_cn3 replicated 1/1 ghcr.io/gnosisguild/ciphernode: ``` # Get the logs diff --git a/docs/pages/ciphernode-operators/running.mdx b/docs/pages/ciphernode-operators/running.mdx index ce0c57f46f..214ef354a0 100644 --- a/docs/pages/ciphernode-operators/running.mdx +++ b/docs/pages/ciphernode-operators/running.mdx @@ -164,8 +164,12 @@ For containerized deployments, you can run the ciphernode Docker image directly. ### Pull the Image +Pull and run an explicit versioned release tag for an operator node. Do not use the mutable `latest` +tag: + ```bash -docker pull ghcr.io/gnosisguild/ciphernode:latest +export CIPHERNODE_IMAGE='ghcr.io/gnosisguild/ciphernode:' +docker pull "$CIPHERNODE_IMAGE" ``` ### Create Configuration @@ -205,17 +209,42 @@ chains: deploy_block: 11124533 ``` +### Create the Credentials Secret + +The image entrypoint requires a JSON secret mounted at `/run/secrets/secrets.json`. It does not read +wallet credentials from `PRIVATE_KEY` or `ENCRYPTION_PASSWORD` environment variables. Create this +file on the host with restrictive permissions; do not commit it, paste it into shell history, or +reuse the example/development keys from the repository: + +```bash +umask 077 +install -m 600 /dev/null secrets.json +${EDITOR:-vi} secrets.json +``` + +```json +{ + "password": "", + "private_key": "", + "network_private_key": "" +} +``` + +All three values are required. Back up the encrypted node data, this password, and the network key +through your secret-management system. Keep the wallet key in the project-approved signing system. + ### Run the Container ```bash docker run -d \ --name ciphernode \ + --restart unless-stopped \ + --stop-timeout 300 \ -v $(pwd)/config.yaml:/home/ciphernode/.config/interfold/config.yaml:ro \ + -v $(pwd)/secrets.json:/run/secrets/secrets.json:ro \ -v ciphernode-data:/home/ciphernode/.local/share/interfold \ -p 9091:9091/udp \ - -e ENCRYPTION_PASSWORD=your_password \ - -e PRIVATE_KEY=0xYourPrivateKey \ - ghcr.io/gnosisguild/ciphernode:latest + "$CIPHERNODE_IMAGE" ``` ### Docker Compose @@ -225,21 +254,41 @@ For a more manageable setup, use Docker Compose: ```yaml services: ciphernode: - image: ghcr.io/gnosisguild/ciphernode:latest + image: ${CIPHERNODE_IMAGE:?CIPHERNODE_IMAGE-must-use-a-versioned-release-tag} restart: unless-stopped + stop_grace_period: 5m volumes: - ./config.yaml:/home/ciphernode/.config/interfold/config.yaml:ro - ciphernode-data:/home/ciphernode/.local/share/interfold + secrets: + - source: ciphernode_credentials + target: secrets.json ports: - '9091:9091/udp' - environment: - ENCRYPTION_PASSWORD: ${ENCRYPTION_PASSWORD} - PRIVATE_KEY: ${PRIVATE_KEY} + +secrets: + ciphernode_credentials: + file: ./secrets.json volumes: ciphernode-data: ``` +Before starting after a crash, restore, or binary upgrade, keep the container stopped and run the +same release's offline integrity check against the mounted configuration and data: + +```bash +docker run --rm \ + -v $(pwd)/config.yaml:/home/ciphernode/.config/interfold/config.yaml:ro \ + -v ciphernode-data:/home/ciphernode/.local/share/interfold \ + --entrypoint interfold \ + "$CIPHERNODE_IMAGE" \ + node validate --config /home/ciphernode/.config/interfold/config.yaml +``` + +Do not start the node when validation reports `FAIL`; preserve a copy of the volume before repair, +restore, or controlled resynchronization. + ### View Logs ```bash From d95e79a4e3999de50a45ae808cd85388bffb0760 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:39:37 +0500 Subject: [PATCH 12/62] fix(zk): bind share proofs to canonical party signers --- .../domain/threshold_plaintext_aggregation.rs | 55 +- .../src/actors/share_verification.rs | 46 +- .../src/domain/share_verification.rs | 672 +++++++++++++++++- 3 files changed, 739 insertions(+), 34 deletions(-) diff --git a/crates/aggregator/src/domain/threshold_plaintext_aggregation.rs b/crates/aggregator/src/domain/threshold_plaintext_aggregation.rs index 0f9b37a986..71f8fde386 100644 --- a/crates/aggregator/src/domain/threshold_plaintext_aggregation.rs +++ b/crates/aggregator/src/domain/threshold_plaintext_aggregation.rs @@ -14,7 +14,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use anyhow::{bail, Result}; +use anyhow::{bail, ensure, Result}; use e3_events::CircuitName; use e3_events::{ DecryptionAggregationJobRequest, PartyProofsToVerify, Proof, Seed, SignedProofPayload, @@ -177,6 +177,17 @@ impl ThresholdPlaintextAggregation { ) -> Result { info!("Adding share for party_id={}", party_id); let current: Collecting = state.try_into()?; + let expected_outputs = current.ciphertext_output.len(); + ensure!( + share.len() == expected_outputs, + "party {party_id} supplied {} decryption shares for {expected_outputs} ciphertext outputs", + share.len() + ); + ensure!( + signed_decryption_proofs.len() == expected_outputs, + "party {party_id} supplied {} C6 proofs for {expected_outputs} ciphertext outputs", + signed_decryption_proofs.len() + ); let ciphertext_output = current.ciphertext_output; let threshold_m = current.threshold_m; let threshold_n = current.threshold_n; @@ -447,6 +458,21 @@ mod tests { ArcBytes::from_bytes(&[b]) } + fn c6_proof(marker: u8) -> SignedProofPayload { + SignedProofPayload { + payload: e3_events::ProofPayload { + e3_id: e3_events::E3id::new("1", 1), + proof_type: e3_events::ProofType::C6ThresholdShareDecryption, + proof: Proof::new( + CircuitName::ThresholdShareDecryption, + ab(marker), + ab(marker), + ), + }, + signature: ab(marker), + } + } + fn collecting(threshold_m: u64, threshold_n: u64) -> ThresholdPlaintextAggregatorState { ThresholdPlaintextAggregatorState::init( threshold_m, @@ -461,7 +487,8 @@ mod tests { fn add_share_below_required_stays_collecting() { let state = collecting(1, 3); let next = - ThresholdPlaintextAggregation::add_share(state, 0, vec![ab(10)], vec![], 3).unwrap(); + ThresholdPlaintextAggregation::add_share(state, 0, vec![ab(10)], vec![c6_proof(10)], 3) + .unwrap(); match next { ThresholdPlaintextAggregatorState::Collecting(c) => { assert_eq!(c.shares.len(), 1); @@ -479,7 +506,7 @@ mod tests { state, pid, vec![ab(pid as u8)], - vec![], + vec![c6_proof(pid as u8)], 3, ) .unwrap(); @@ -514,7 +541,7 @@ mod tests { state, pid, vec![ab(pid as u8)], - vec![], + vec![c6_proof(pid as u8)], 3, ) .unwrap(); @@ -538,7 +565,7 @@ mod tests { state, pid, vec![ab(pid as u8)], - vec![], + vec![c6_proof(pid as u8)], 3, ) .unwrap(); @@ -587,6 +614,24 @@ mod tests { )); } + #[test] + fn add_share_rejects_c6_share_or_proof_count_mismatch() { + let missing_share = ThresholdPlaintextAggregation::add_share( + collecting(1, 3), + 0, + vec![], + vec![c6_proof(1)], + 3, + ) + .expect_err("one decryption share is required for one ciphertext"); + assert!(missing_share.to_string().contains("decryption shares")); + + let missing_proof = + ThresholdPlaintextAggregation::add_share(collecting(1, 3), 0, vec![ab(1)], vec![], 3) + .expect_err("one C6 proof is required for one ciphertext"); + assert!(missing_proof.to_string().contains("C6 proofs")); + } + #[test] fn plan_c6_dispatch_emits_party_proofs_in_party_order() { let mut c6: BTreeMap> = BTreeMap::new(); diff --git a/crates/zk-prover/src/actors/share_verification.rs b/crates/zk-prover/src/actors/share_verification.rs index 935767122b..a37b20cbd5 100644 --- a/crates/zk-prover/src/actors/share_verification.rs +++ b/crates/zk-prover/src/actors/share_verification.rs @@ -55,6 +55,9 @@ use crate::domain::share_verification::{ /// [`ShareVerificationComplete`] with the final dishonest party set. pub struct ShareVerificationActor { bus: BusHandle, + /// Canonical finalized committees in party-id order. Every signed C1-C4/C6 proof must recover + /// to the address that owns its outer `sender_party_id` slot. + committees: HashMap>, /// Tracks pending ZK verifications by correlation ID. pending: HashMap, /// Tracks pending consistency checks by correlation ID (between ECDSA and ZK). @@ -65,6 +68,7 @@ impl ShareVerificationActor { pub fn new(bus: &BusHandle) -> Self { Self { bus: bus.clone(), + committees: HashMap::new(), pending: HashMap::new(), pending_consistency: HashMap::new(), } @@ -72,6 +76,7 @@ impl ShareVerificationActor { pub fn setup(bus: &BusHandle) -> Addr { let addr = Self::new(bus).start(); + bus.subscribe(EventType::CommitteeFinalized, addr.clone().into()); bus.subscribe(EventType::ShareVerificationDispatched, addr.clone().into()); bus.subscribe(EventType::ComputeResponse, addr.clone().into()); bus.subscribe(EventType::ComputeRequestError, addr.clone().into()); @@ -79,6 +84,7 @@ impl ShareVerificationActor { EventType::CommitmentConsistencyCheckComplete, addr.clone().into(), ); + bus.subscribe(EventType::E3RequestComplete, addr.clone().into()); addr } @@ -154,7 +160,16 @@ impl ShareVerificationActor { // Pure ECDSA validation + proof-commitment preparation lives in the // domain service; the actor only emits failures, stores pending state, // and publishes the consistency-check request. - let outcome = ShareVerifier::validate_and_prepare(&party_proofs, &e3_id_str, label); + let committee = self.committees.get(&e3_id).map(Vec::as_slice); + let outcome = ShareVerifier::validate_and_prepare( + &party_proofs, + &e3_id_str, + &kind, + label, + committee, + params_preset, + committee_size, + ); for failure in &outcome.failures { self.emit_signed_proof_failed( @@ -576,6 +591,28 @@ impl Handler for ShareVerificationActor { fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { let (msg, ec) = msg.into_components(); match msg { + InterfoldEventData::CommitteeFinalized(mut data) => { + // Mirror the C0 verifier's canonical ordering at this trust boundary. Replayed and + // test-produced events are not assumed to have passed through the EVM decoder. + data.sort_by_score(); + match data + .committee + .iter() + .map(|member| member.parse()) + .collect::, _>>() + { + Ok(committee) => { + self.committees.insert(data.e3_id, committee); + } + Err(error) => { + error!( + e3_id = %data.e3_id, + %error, + "Finalized committee contains an invalid address; share proofs will be rejected" + ); + } + } + } InterfoldEventData::ShareVerificationDispatched(data) => { self.notify_sync(ctx, TypedEvent::new(data, ec)) } @@ -588,6 +625,13 @@ impl Handler for ShareVerificationActor { InterfoldEventData::CommitmentConsistencyCheckComplete(data) => { self.notify_sync(ctx, TypedEvent::new(data, ec)) } + InterfoldEventData::E3RequestComplete(data) => { + let e3_id = data.e3_id; + self.committees.remove(&e3_id); + self.pending.retain(|_, pending| pending.e3_id != e3_id); + self.pending_consistency + .retain(|_, pending| pending.e3_id != e3_id); + } _ => (), } } diff --git a/crates/zk-prover/src/domain/share_verification.rs b/crates/zk-prover/src/domain/share_verification.rs index dae89a1754..f9b50f5fc0 100644 --- a/crates/zk-prover/src/domain/share_verification.rs +++ b/crates/zk-prover/src/domain/share_verification.rs @@ -26,7 +26,7 @@ use e3_zk_helpers::CiphernodesCommitteeSize; use tracing::{info, warn}; /// Trait for party types whose signed proofs can be ECDSA-validated and ZK-verified. -pub(crate) trait VerifiableParty: Clone { +pub(crate) trait VerifiableParty: Clone + PartialEq { fn party_id(&self) -> u64; fn signed_proofs(&self) -> Vec; } @@ -194,17 +194,95 @@ impl ShareVerifier { keccak256(&msg).into() } + /// Check that a party supplied the canonical proof-type layout for this protocol phase. + /// + /// C2/C3 counts are derived from the DKG parameter preset. Variable C4b and C6 counts are + /// checked against trusted local state by their producers. This trust-boundary check prevents + /// a signed proof for another phase (or a duplicated singleton proof) from satisfying the + /// current phase merely because its self-declared [`ProofType`] maps to a valid circuit. + fn has_canonical_proof_shape( + kind: &VerificationKind, + signed_proofs: &[SignedProofPayload], + params_preset: e3_fhe_params::BfvPreset, + ) -> bool { + match kind { + VerificationKind::PkGenerationProofs => { + signed_proofs.len() == 1 + && signed_proofs[0].payload.proof_type == ProofType::C1PkGeneration + } + VerificationKind::ShareProofs => { + // Canonical order is C2a, C2b, C3a x L, C3b x L. Share encryption uses the + // DKG counterpart of a threshold preset (or the preset itself when already DKG). + let dkg_preset = params_preset.dkg_counterpart().unwrap_or(params_preset); + let num_moduli = dkg_preset.metadata().num_moduli; + signed_proofs.len() == 2 + (2 * num_moduli) + && signed_proofs[0].payload.proof_type == ProofType::C2aSkShareComputation + && signed_proofs[1].payload.proof_type == ProofType::C2bESmShareComputation + && signed_proofs[2..2 + num_moduli] + .iter() + .all(|signed| signed.payload.proof_type == ProofType::C3aSkShareEncryption) + && signed_proofs[2 + num_moduli..] + .iter() + .all(|signed| signed.payload.proof_type == ProofType::C3bESmShareEncryption) + } + VerificationKind::DecryptionProofs => { + // PartyShareDecryptionProofsToVerify has one distinguished C4a slot followed + // by one or more C4b slots. The producer checks the exact C4b count against + // `es_poly_sum`; here we bind every signed payload to its structural role because + // C4a/C4b share a CircuitName. + signed_proofs.len() >= 2 + && signed_proofs[0].payload.proof_type == ProofType::C4aSkShareDecryption + && signed_proofs[1..] + .iter() + .all(|signed| signed.payload.proof_type == ProofType::C4bESmShareDecryption) + } + VerificationKind::ThresholdDecryptionProofs => { + !signed_proofs.is_empty() + && signed_proofs.iter().all(|signed| { + signed.payload.proof_type == ProofType::C6ThresholdShareDecryption + }) + } + } + } + /// Validate ECDSA properties for a set of signed proofs from one party: /// 1. e3_id match /// 2. Signature recovery (valid ECDSA) - /// 3. Signer consistency (all proofs from same address) - /// 4. Circuit name matches expected ProofType circuits + /// 3. Recovered signer owns the canonical finalized-committee party slot + /// 4. Signer consistency (all proofs from same address) + /// 5. Circuit name matches expected ProofType circuits pub(crate) fn ecdsa_validate_signed_proofs( sender_party_id: u64, signed_proofs: &[SignedProofPayload], e3_id_str: &str, label: &str, + expected_signer: Option

, ) -> EcdsaPartyResult { + if signed_proofs.is_empty() { + info!( + "{} party {} supplied an empty signed-proof bundle", + label, sender_party_id + ); + return EcdsaPartyResult { + passed: false, + failed_payload: None, + }; + } + + let Some(expected_signer) = expected_signer else { + info!( + "{} party {} has no canonical finalized-committee slot", + label, sender_party_id + ); + return EcdsaPartyResult { + passed: false, + // The outer party id is not part of the signed payload. Its absence from the + // canonical committee is therefore a structural dispatch failure, not + // self-authenticating evidence that can safely be attributed to the signer. + failed_payload: None, + }; + }; + let mut expected_addr: Option
= None; for signed in signed_proofs { @@ -223,7 +301,17 @@ impl ShareVerifier { // 2. Signature recovery match signed.recover_address() { Ok(addr) => { - // 3. Signer consistency + // 3. Canonical party ownership and signer consistency + if addr != expected_signer { + info!( + "{} proof signer {} does not own party {} (expected {})", + label, addr, sender_party_id, expected_signer + ); + return EcdsaPartyResult { + passed: false, + failed_payload: Some((signed.clone(), Some(addr))), + }; + } match &expected_addr { Some(ea) if *ea != addr => { info!( @@ -277,17 +365,102 @@ impl ShareVerifier { pub(crate) fn validate_and_prepare( party_proofs: &[P], e3_id_str: &str, + kind: &VerificationKind, label: &str, + committee: Option<&[Address]>, + params_preset: e3_fhe_params::BfvPreset, + committee_size: CiphernodesCommitteeSize, ) -> EcdsaValidationOutcome

{ let mut ecdsa_dishonest = HashSet::new(); let mut failures = Vec::new(); let mut ecdsa_passed_parties = Vec::new(); let mut party_addresses: HashMap = HashMap::new(); + // A finalized committee is a one-to-one party-slot map. The on-chain sortition path + // guarantees unique operators, but replayed/test-produced events also cross this + // boundary. Refuse an ambiguous map rather than allowing one signer to own two slots. + let expected_committee_n = committee_size.values().n; + let committee = committee.filter(|members| { + let unique_members: HashSet

= members.iter().copied().collect(); + let is_unique = unique_members.len() == members.len(); + let has_expected_size = members.len() == expected_committee_n; + if !is_unique { + warn!( + "{} finalized committee contains duplicate addresses; rejecting proof batch", + label + ); + } + if !has_expected_size { + warn!( + "{} finalized committee has {} members, expected {}; rejecting proof batch", + label, + members.len(), + expected_committee_n + ); + } + is_unique && has_expected_size + }); + + // Collapse byte-identical replayed bundles idempotently. Two different bundles for the + // same outer party id are ambiguous and are both excluded, so a canonical party can + // contribute at most one verification result and one set of passed-proof emissions. + let mut unique_parties: Vec<&P> = Vec::new(); + let mut party_positions: HashMap = HashMap::new(); + let mut conflicting_party_ids = HashSet::new(); for party in party_proofs { + match party_positions.get(&party.party_id()).copied() { + Some(position) if unique_parties[position] == party => { + warn!( + "{} duplicate replay for party {} ignored", + label, + party.party_id() + ); + } + Some(_) => { + warn!( + "{} conflicting proof bundles for party {}; rejecting that party", + label, + party.party_id() + ); + conflicting_party_ids.insert(party.party_id()); + } + None => { + party_positions.insert(party.party_id(), unique_parties.len()); + unique_parties.push(party); + } + } + } + + for party in unique_parties { + if conflicting_party_ids.contains(&party.party_id()) { + ecdsa_dishonest.insert(party.party_id()); + continue; + } + let proofs = party.signed_proofs(); - let result = - Self::ecdsa_validate_signed_proofs(party.party_id(), &proofs, e3_id_str, label); + if !Self::has_canonical_proof_shape(kind, &proofs, params_preset) { + info!( + "{} party {} supplied a non-canonical proof-type multiplicity/order", + label, + party.party_id() + ); + // The verification kind and outer bundle are not signed. Exclude the party, + // but do not manufacture slash evidence from an otherwise valid signed proof. + ecdsa_dishonest.insert(party.party_id()); + continue; + } + + let expected_signer = usize::try_from(party.party_id()) + .ok() + .and_then(|party_id| committee.and_then(|members| members.get(party_id))) + .copied(); + let result = Self::ecdsa_validate_signed_proofs( + party.party_id(), + &proofs, + e3_id_str, + label, + expected_signer, + ); if result.passed { ecdsa_passed_parties.push(party.clone()); } else { @@ -303,13 +476,11 @@ impl ShareVerifier { } // Store recovered addresses for passed parties. - for party in party_proofs { - if !ecdsa_dishonest.contains(&party.party_id()) { - let proofs = party.signed_proofs(); - if let Some(first_signed) = proofs.first() { - if let Ok(addr) = first_signed.recover_address() { - party_addresses.insert(party.party_id(), addr); - } + for party in &ecdsa_passed_parties { + let proofs = party.signed_proofs(); + if let Some(first_signed) = proofs.first() { + if let Ok(addr) = first_signed.recover_address() { + party_addresses.insert(party.party_id(), addr); } } } @@ -401,11 +572,45 @@ impl ShareVerifier { let mut dishonest: BTreeSet = pre_dishonest; dishonest.extend(ecdsa_dishonest); - // Cross-check: every dispatched party must appear in results. - let returned_party_ids: HashSet = - zk_results.iter().map(|r| r.sender_party_id).collect(); + // Canonicalize the worker response before tallying. Exact duplicate responses are + // harmless replays and are collapsed; conflicting responses for one party fail closed. + // This also guarantees at most one Passed/Failed emission per canonical party. + let mut unique_results: HashMap = HashMap::new(); + let mut result_order = Vec::new(); + let mut conflicting_party_ids = HashSet::new(); + for result in zk_results { + if !dispatched_party_ids.contains(&result.sender_party_id) { + warn!( + "ZK result for party {} was not dispatched — ignoring", + result.sender_party_id + ); + continue; + } + + match unique_results.get(&result.sender_party_id).copied() { + Some(existing) if existing == result => { + warn!( + "Duplicate ZK result replay for party {} ignored", + result.sender_party_id + ); + } + Some(_) => { + warn!( + "Conflicting ZK results for party {} — treating as dishonest", + result.sender_party_id + ); + conflicting_party_ids.insert(result.sender_party_id); + } + None => { + unique_results.insert(result.sender_party_id, result); + result_order.push(result.sender_party_id); + } + } + } + + // Cross-check: every dispatched party must appear exactly once after replay collapse. for &dispatched_pid in dispatched_party_ids { - if !returned_party_ids.contains(&dispatched_pid) { + if !unique_results.contains_key(&dispatched_pid) { warn!( "Party {} was dispatched for ZK verification but missing from results — treating as dishonest", dispatched_pid @@ -415,15 +620,17 @@ impl ShareVerifier { } let mut emissions = Vec::new(); - for result in zk_results { - // Ignore results for parties we never dispatched (defense-in-depth). - if !dispatched_party_ids.contains(&result.sender_party_id) { - warn!( - "ZK result for party {} was not dispatched — ignoring", - result.sender_party_id - ); + for party_id in result_order { + if conflicting_party_ids.contains(&party_id) { + dishonest.insert(party_id); continue; } + let Some(result) = unique_results.get(&party_id) else { + // `result_order` and `unique_results` are populated together above. Keep this + // boundary panic-free if that implementation changes later. + dishonest.insert(party_id); + continue; + }; if !result.all_verified { dishonest.insert(result.sender_party_id); if let Some(ref signed) = result.failed_signed_payload { @@ -451,16 +658,49 @@ mod tests { use super::*; use alloy::signers::local::PrivateKeySigner; use e3_events::{Proof, ProofPayload, ProofType}; + use e3_fhe_params::BfvPreset; fn signer() -> PrivateKeySigner { PrivateKeySigner::random() } + fn minimum_committee(mut members: Vec
) -> Vec
{ + while members.len() < CiphernodesCommitteeSize::Minimum.values().n { + let candidate = signer().address(); + if !members.contains(&candidate) { + members.push(candidate); + } + } + members + } + + fn signed_proof( + s: &PrivateKeySigner, + e3_id: &E3id, + proof_type: ProofType, + marker: u8, + ) -> SignedProofPayload { + let proof = Proof::new( + proof_type.circuit_names()[0], + ArcBytes::from_bytes(&[marker, 2, 3]), + ArcBytes::from_bytes(&[4, 5, marker]), + ); + let payload = ProofPayload { + e3_id: e3_id.clone(), + proof_type, + proof, + }; + SignedProofPayload::sign(payload, s).expect("sign") + } + /// Build a signed C1 (PkGeneration) proof for `party_id` under `e3_id`, /// optionally with a deliberately wrong circuit name. fn signed_pk(s: &PrivateKeySigner, e3_id: &E3id, wrong_circuit: bool) -> SignedProofPayload { use e3_events::CircuitName; let proof_type = ProofType::C1PkGeneration; + if !wrong_circuit { + return signed_proof(s, e3_id, proof_type, 1); + } let circuit = if wrong_circuit { CircuitName::PkBfv } else { @@ -488,7 +728,13 @@ mod tests { let s = signer(); let e3 = e3(); let p = signed_pk(&s, &e3, false); - let res = ShareVerifier::ecdsa_validate_signed_proofs(7, &[p], &e3.to_string(), "C1"); + let res = ShareVerifier::ecdsa_validate_signed_proofs( + 7, + &[p], + &e3.to_string(), + "C1", + Some(s.address()), + ); assert!(res.passed); assert!(res.failed_payload.is_none()); } @@ -497,7 +743,8 @@ mod tests { fn ecdsa_fails_on_wrong_e3_id() { let s = signer(); let p = signed_pk(&s, &e3(), false); - let res = ShareVerifier::ecdsa_validate_signed_proofs(7, &[p], "999/0", "C1"); + let res = + ShareVerifier::ecdsa_validate_signed_proofs(7, &[p], "999/0", "C1", Some(s.address())); assert!(!res.passed); assert!(res.failed_payload.is_some()); } @@ -507,7 +754,13 @@ mod tests { let s = signer(); let e3 = e3(); let p = signed_pk(&s, &e3, true); - let res = ShareVerifier::ecdsa_validate_signed_proofs(7, &[p], &e3.to_string(), "C1"); + let res = ShareVerifier::ecdsa_validate_signed_proofs( + 7, + &[p], + &e3.to_string(), + "C1", + Some(s.address()), + ); assert!(!res.passed); } @@ -518,10 +771,323 @@ mod tests { let e3 = e3(); let p1 = signed_pk(&s1, &e3, false); let p2 = signed_pk(&s2, &e3, false); - let res = ShareVerifier::ecdsa_validate_signed_proofs(7, &[p1, p2], &e3.to_string(), "C1"); + let res = ShareVerifier::ecdsa_validate_signed_proofs( + 7, + &[p1, p2], + &e3.to_string(), + "C1", + Some(s1.address()), + ); assert!(!res.passed); } + #[test] + fn ecdsa_fails_when_signer_does_not_own_party_slot() { + let proof_signer = signer(); + let slot_owner = signer(); + let e3 = e3(); + let proof = signed_pk(&proof_signer, &e3, false); + + let result = ShareVerifier::ecdsa_validate_signed_proofs( + 1, + &[proof], + &e3.to_string(), + "C1", + Some(slot_owner.address()), + ); + + assert!(!result.passed); + let (_, recovered) = result.failed_payload.expect("attributable mismatch"); + assert_eq!(recovered, Some(proof_signer.address())); + } + + #[test] + fn ecdsa_fails_for_empty_bundle_or_missing_party_slot() { + let e3 = e3(); + let owner = signer(); + let empty = ShareVerifier::ecdsa_validate_signed_proofs( + 0, + &[], + &e3.to_string(), + "C1", + Some(owner.address()), + ); + assert!(!empty.passed); + assert!(empty.failed_payload.is_none()); + + let proof = signed_pk(&owner, &e3, false); + let missing = + ShareVerifier::ecdsa_validate_signed_proofs(3, &[proof], &e3.to_string(), "C1", None); + assert!(!missing.passed); + } + + #[test] + fn prepare_rejects_one_signer_relabelled_across_other_party_slots() { + let first = signer(); + let second = signer(); + let third = signer(); + let e3 = e3(); + let parties = vec![ + PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_pk(&first, &e3, false)], + }, + PartyProofsToVerify { + sender_party_id: 1, + // A valid proof from party 0 cannot fill party 1's slot. + signed_proofs: vec![signed_pk(&first, &e3, false)], + }, + PartyProofsToVerify { + sender_party_id: 2, + // Nor can the same signer fill any other canonical slot. + signed_proofs: vec![signed_pk(&first, &e3, false)], + }, + ]; + let committee = [first.address(), second.address(), third.address()]; + + let outcome = ShareVerifier::validate_and_prepare( + &parties, + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + + assert_eq!(outcome.ecdsa_passed_parties.len(), 1); + assert_eq!(outcome.ecdsa_passed_parties[0].sender_party_id, 0); + assert_eq!(outcome.ecdsa_dishonest, HashSet::from([1, 2])); + } + + #[test] + fn canonical_shape_rejects_cross_phase_and_singleton_multiplicity() { + let s = signer(); + let e3 = e3(); + + let c1 = signed_proof(&s, &e3, ProofType::C1PkGeneration, 1); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::PkGenerationProofs, + std::slice::from_ref(&c1), + BfvPreset::InsecureDkg512, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::PkGenerationProofs, + &[c1.clone(), c1.clone()], + BfvPreset::InsecureDkg512, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ThresholdDecryptionProofs, + std::slice::from_ref(&c1), + BfvPreset::InsecureDkg512, + )); + let c6 = signed_proof(&s, &e3, ProofType::C6ThresholdShareDecryption, 9); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ThresholdDecryptionProofs, + std::slice::from_ref(&c6), + BfvPreset::InsecureDkg512, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ThresholdDecryptionProofs, + &[], + BfvPreset::InsecureDkg512, + )); + + let share_bundle = vec![ + signed_proof(&s, &e3, ProofType::C2aSkShareComputation, 2), + signed_proof(&s, &e3, ProofType::C2bESmShareComputation, 3), + signed_proof(&s, &e3, ProofType::C3aSkShareEncryption, 4), + signed_proof(&s, &e3, ProofType::C3bESmShareEncryption, 5), + ]; + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &share_bundle, + BfvPreset::InsecureDkg512, + )); + let mut duplicate_c2a = share_bundle.clone(); + duplicate_c2a.insert(1, duplicate_c2a[0].clone()); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &duplicate_c2a, + BfvPreset::InsecureDkg512, + )); + + let secure_share_bundle = vec![ + signed_proof(&s, &e3, ProofType::C2aSkShareComputation, 10), + signed_proof(&s, &e3, ProofType::C2bESmShareComputation, 11), + signed_proof(&s, &e3, ProofType::C3aSkShareEncryption, 12), + signed_proof(&s, &e3, ProofType::C3aSkShareEncryption, 13), + signed_proof(&s, &e3, ProofType::C3bESmShareEncryption, 14), + signed_proof(&s, &e3, ProofType::C3bESmShareEncryption, 15), + ]; + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &secure_share_bundle, + BfvPreset::SecureThreshold8192, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &share_bundle, + BfvPreset::SecureThreshold8192, + )); + + let c4_bundle = vec![ + signed_proof(&s, &e3, ProofType::C4aSkShareDecryption, 6), + signed_proof(&s, &e3, ProofType::C4bESmShareDecryption, 7), + ]; + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::DecryptionProofs, + &c4_bundle, + BfvPreset::InsecureDkg512, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::DecryptionProofs, + &c4_bundle[1..], + BfvPreset::InsecureDkg512, + )); + let mut extra_c4b = c4_bundle.clone(); + extra_c4b.push(c4_bundle[1].clone()); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::DecryptionProofs, + &extra_c4b, + BfvPreset::InsecureDkg512, + )); + let mut wrong_c4_tail = c4_bundle.clone(); + wrong_c4_tail.push(c6); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::DecryptionProofs, + &wrong_c4_tail, + BfvPreset::InsecureDkg512, + )); + } + + #[test] + fn prepare_excludes_wrong_phase_without_creating_slash_evidence() { + let s = signer(); + let e3 = e3(); + let parties = [PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_proof( + &s, + &e3, + ProofType::C6ThresholdShareDecryption, + 9, + )], + }]; + let committee = minimum_committee(vec![s.address()]); + + let outcome = ShareVerifier::validate_and_prepare( + &parties, + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + + assert!(outcome.ecdsa_passed_parties.is_empty()); + assert_eq!(outcome.ecdsa_dishonest, HashSet::from([0])); + assert!(outcome.failures.is_empty()); + } + + #[test] + fn prepare_collapses_identical_party_replay_and_rejects_conflict() { + let s = signer(); + let e3 = e3(); + let committee = minimum_committee(vec![s.address()]); + let party = PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 1)], + }; + + let replayed = ShareVerifier::validate_and_prepare( + &[party.clone(), party.clone()], + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + assert_eq!(replayed.ecdsa_passed_parties, vec![party.clone()]); + assert!(replayed.ecdsa_dishonest.is_empty()); + assert_eq!(replayed.consistency_party_data.len(), 1); + + let conflicting = PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 2)], + }; + let conflict = ShareVerifier::validate_and_prepare( + &[party, conflicting], + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + assert!(conflict.ecdsa_passed_parties.is_empty()); + assert_eq!(conflict.ecdsa_dishonest, HashSet::from([0])); + assert!(conflict.failures.is_empty()); + } + + #[test] + fn prepare_rejects_ambiguous_committee_where_one_signer_owns_multiple_slots() { + let s = signer(); + let e3 = e3(); + let parties = vec![ + PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 1)], + }, + PartyProofsToVerify { + sender_party_id: 1, + signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 2)], + }, + ]; + let ambiguous_committee = [s.address(), s.address(), signer().address()]; + + let outcome = ShareVerifier::validate_and_prepare( + &parties, + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&ambiguous_committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + + assert!(outcome.ecdsa_passed_parties.is_empty()); + assert_eq!(outcome.ecdsa_dishonest, HashSet::from([0, 1])); + assert!(outcome.failures.is_empty()); + } + + #[test] + fn prepare_rejects_committee_with_wrong_circuit_dimension() { + let s = signer(); + let e3 = e3(); + let parties = [PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 1)], + }]; + let undersized_committee = [s.address()]; + + let outcome = ShareVerifier::validate_and_prepare( + &parties, + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&undersized_committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + + assert!(outcome.ecdsa_passed_parties.is_empty()); + assert_eq!(outcome.ecdsa_dishonest, HashSet::from([0])); + assert!(outcome.failures.is_empty()); + } + #[test] fn filter_consistent_drops_inconsistent_and_returns_ids() { let proofs = vec![1u64, 2, 3]; @@ -548,4 +1114,54 @@ mod tests { assert!(out.dishonest.contains(&2)); assert!(out.emissions.is_empty()); } + + #[test] + fn tally_collapses_identical_result_replay() { + let dispatched = HashSet::from([1]); + let result = PartyVerificationResult { + sender_party_id: 1, + all_verified: true, + failed_signed_payload: None, + recovered_address: None, + }; + + let out = ShareVerifier::tally_zk_results( + BTreeSet::new(), + &HashSet::new(), + &dispatched, + &[result.clone(), result], + ); + + assert!(out.dishonest.is_empty()); + assert_eq!(out.emissions.len(), 1); + assert!(matches!( + out.emissions[0], + ZkPartyEmission::Passed { party_id: 1 } + )); + } + + #[test] + fn tally_rejects_conflicting_results_for_one_party() { + let dispatched = HashSet::from([1]); + let passed = PartyVerificationResult { + sender_party_id: 1, + all_verified: true, + failed_signed_payload: None, + recovered_address: None, + }; + let failed = PartyVerificationResult { + all_verified: false, + ..passed.clone() + }; + + let out = ShareVerifier::tally_zk_results( + BTreeSet::new(), + &HashSet::new(), + &dispatched, + &[passed, failed], + ); + + assert_eq!(out.dishonest, BTreeSet::from([1])); + assert!(out.emissions.is_empty()); + } } From 52117c46aa4117879af12f781403e94f907ac816 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:43:08 +0500 Subject: [PATCH 13/62] fix(aggregation): bind share senders to party slots --- .../actors/keyshare_created_filter_buffer.rs | 59 +++++++++++++-- .../src/actors/publickey_aggregator.rs | 1 - .../actors/threshold_plaintext_aggregator.rs | 75 ++++++++++++++++--- 3 files changed, 118 insertions(+), 17 deletions(-) diff --git a/crates/aggregator/src/actors/keyshare_created_filter_buffer.rs b/crates/aggregator/src/actors/keyshare_created_filter_buffer.rs index e8713b95ca..8a608eda50 100644 --- a/crates/aggregator/src/actors/keyshare_created_filter_buffer.rs +++ b/crates/aggregator/src/actors/keyshare_created_filter_buffer.rs @@ -5,6 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::prelude::*; +use alloy::primitives::Address; use e3_events::{prelude::*, AggregatorChanged, Die, InterfoldEvent, InterfoldEventData}; use e3_utils::MAILBOX_LIMIT; @@ -16,12 +17,26 @@ use crate::PublicKeyAggregator; /// Buffers `KeyshareCreated` events until `CommitteeFinalized` arrives. pub struct KeyshareCreatedFilterBuffer { dest: Addr, - committee: Option>, + /// Finalized committee in canonical party-id order. A membership set is insufficient here: + /// accepting a real member under another member's `party_id` poisons the DKG roster. + committee: Option>, buffer: Vec, expelled_nodes: HashSet, is_aggregator: bool, } +fn committee_member_matches(committee: &[String], party_id: u64, node: &str) -> bool { + let Some(expected) = usize::try_from(party_id) + .ok() + .and_then(|index| committee.get(index)) + .and_then(|member| member.parse::
().ok()) + else { + return false; + }; + node.parse::
() + .is_ok_and(|address| address == expected) +} + impl KeyshareCreatedFilterBuffer { pub fn new(dest: Addr) -> Self { Self { @@ -42,7 +57,7 @@ impl KeyshareCreatedFilterBuffer { for event in self.buffer.drain(..) { match event.get_data() { InterfoldEventData::KeyshareCreated(data) - if committee.contains(&data.node) + if committee_member_matches(committee, data.party_id, &data.node) && !self.expelled_nodes.contains(&data.node) => { self.dest.do_send(event); @@ -77,7 +92,7 @@ impl Handler for KeyshareCreatedFilterBuffer { InterfoldEventData::KeyshareCreated(data) => match &self.committee { Some(committee) if self.is_aggregator - && committee.contains(&data.node) + && committee_member_matches(committee, data.party_id, &data.node) && !self.expelled_nodes.contains(&data.node) => { self.dest.do_send(msg); @@ -86,7 +101,7 @@ impl Handler for KeyshareCreatedFilterBuffer { self.buffer.push(msg); } Some(committee) - if committee.contains(&data.node) + if committee_member_matches(committee, data.party_id, &data.node) && !self.expelled_nodes.contains(&data.node) => { self.buffer.push(msg); @@ -94,7 +109,9 @@ impl Handler for KeyshareCreatedFilterBuffer { _ => {} }, InterfoldEventData::CommitteeFinalized(data) => { - self.committee = Some(data.committee.iter().cloned().collect()); + let mut data = data.clone(); + data.sort_by_score(); + self.committee = Some(data.committee); self.process_buffered_events(); } InterfoldEventData::CommitteeMemberExpelled(data) => { @@ -111,12 +128,11 @@ impl Handler for KeyshareCreatedFilterBuffer { ) }); - if let Some(ref mut committee) = self.committee { + if self.committee.is_some() { info!( "KeyshareCreatedFilterBuffer: removing expelled node {} from committee filter (e3_id={})", node_addr, data.e3_id ); - committee.remove(&node_addr); } if self.is_aggregator { @@ -147,3 +163,32 @@ impl Handler for KeyshareCreatedFilterBuffer { ctx.stop(); } } + +#[cfg(test)] +mod tests { + use super::committee_member_matches; + + const A: &str = "0x1111111111111111111111111111111111111111"; + const B: &str = "0x2222222222222222222222222222222222222222"; + + #[test] + fn committee_membership_is_bound_to_party_slot() { + let committee = vec![A.to_owned(), B.to_owned()]; + assert!(committee_member_matches(&committee, 0, A)); + assert!(committee_member_matches(&committee, 1, B)); + assert!(!committee_member_matches(&committee, 0, B)); + assert!(!committee_member_matches(&committee, 1, A)); + assert!(!committee_member_matches(&committee, 2, A)); + assert!(!committee_member_matches(&committee, 0, "not-an-address")); + } + + #[test] + fn committee_address_comparison_is_case_insensitive() { + let committee = vec!["0xAbCd000000000000000000000000000000000000".to_owned()]; + assert!(committee_member_matches( + &committee, + 0, + "0xabcd000000000000000000000000000000000000" + )); + } +} diff --git a/crates/aggregator/src/actors/publickey_aggregator.rs b/crates/aggregator/src/actors/publickey_aggregator.rs index 9b68fb3860..b2a8e060b3 100644 --- a/crates/aggregator/src/actors/publickey_aggregator.rs +++ b/crates/aggregator/src/actors/publickey_aggregator.rs @@ -324,7 +324,6 @@ impl PublicKeyAggregator { .map(|(pid, node, _)| (*pid, node.clone())) .collect(); - let circuit_committee_n = circuit_committee_n; let circuit_committee_h = circuit_h; self.state.try_mutate(&ec, |_| { Ok(PublicKeyAggregatorState::GeneratingC5Proof { diff --git a/crates/aggregator/src/actors/threshold_plaintext_aggregator.rs b/crates/aggregator/src/actors/threshold_plaintext_aggregator.rs index 4b90e906c0..613dcf456e 100644 --- a/crates/aggregator/src/actors/threshold_plaintext_aggregator.rs +++ b/crates/aggregator/src/actors/threshold_plaintext_aggregator.rs @@ -122,6 +122,23 @@ pub struct ThresholdPlaintextAggregatorParams { pub honest_committee_addresses: Vec
, } +fn node_owns_committee_party_slot( + committee: &[Address], + honest_committee: &[Address], + node: &str, + party_id: u64, +) -> bool { + let Some(expected) = usize::try_from(party_id) + .ok() + .and_then(|index| committee.get(index)) + else { + return false; + }; + Address::from_str(node) + .ok() + .is_some_and(|address| &address == expected && honest_committee.contains(&address)) +} + impl ThresholdPlaintextAggregator { pub fn new( params: ThresholdPlaintextAggregatorParams, @@ -155,11 +172,16 @@ impl ThresholdPlaintextAggregator { self.honest_committee_addresses.len() as u64 } - /// True when `node` is in `PublicKeyAggregated.honest_committee_addresses`. - fn node_in_aggregated_pk_committee(&self, node: &str) -> bool { - Address::from_str(node) - .ok() - .is_some_and(|addr| self.honest_committee_addresses.contains(&addr)) + /// True when `node` owns `party_id` in the full canonical committee and is part of the honest + /// subset selected during DKG. Membership without the slot check permits a real member to + /// relabel a share under another party ID. + fn node_owns_aggregated_pk_party_slot(&self, node: &str, party_id: u64) -> bool { + node_owns_committee_party_slot( + &self.committee_addresses, + &self.honest_committee_addresses, + node, + party_id, + ) } pub fn add_share( @@ -319,7 +341,8 @@ impl ThresholdPlaintextAggregator { // failure leaves us in VerifyingC6 (retryable) rather than // Computing (no retry path). // TrBFV scheme size stays N (`threshold_n`); only the share roster is restricted to the - // H canonical honest parties in `PublicKeyAggregated` (see `node_in_aggregated_pk_committee`). + // H canonical honest parties in `PublicKeyAggregated` (see + // `node_owns_aggregated_pk_party_slot`). let trbfv_config = TrBFVConfig::new(state.params.clone(), state.threshold_n, state.threshold_m); @@ -865,10 +888,11 @@ impl Handler>> trace!("Node {} not found in finalized committee", &msg.node); return Ok(()); }; - if !self.node_in_aggregated_pk_committee(&msg.node) { + if !self.node_owns_aggregated_pk_party_slot(&msg.node, msg.party_id) { trace!( - "Node {} not in PublicKeyAggregated honest subset — ignoring decryption share", - &msg.node + "Node {} does not own honest party slot {} — ignoring decryption share", + &msg.node, + msg.party_id ); return Ok(()); } @@ -1042,6 +1066,39 @@ mod tests { ) } + #[test] + fn decryption_sender_must_own_an_honest_party_slot() { + let first = Address::repeat_byte(0x11); + let second = Address::repeat_byte(0x22); + let committee = [first, second]; + let honest = [first]; + + assert!(node_owns_committee_party_slot( + &committee, + &honest, + &first.to_string(), + 0 + )); + assert!(!node_owns_committee_party_slot( + &committee, + &honest, + &first.to_string(), + 1 + )); + assert!(!node_owns_committee_party_slot( + &committee, + &honest, + &second.to_string(), + 1 + )); + assert!(!node_owns_committee_party_slot( + &committee, + &honest, + &first.to_string(), + 2 + )); + } + fn computing_state() -> ThresholdPlaintextAggregatorState { ThresholdPlaintextAggregatorState::Computing(Computing { threshold_m: 1, From 7f4f29024628086ec8d020bed2287f86d48bc369 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:51:55 +0500 Subject: [PATCH 14/62] fix(evm): fail closed on bounded startup buffers --- Cargo.lock | 1 + crates/ciphernode-builder/Cargo.toml | 1 + .../src/ciphernode_builder.rs | 65 +++++-- crates/ciphernode-builder/src/evm_system.rs | 21 ++- crates/config/src/app_config.rs | 42 +++++ crates/entrypoint/src/start/start.rs | 1 + crates/evm/src/actors/evm_chain_gateway.rs | 177 ++++++++++++++++-- crates/evm/src/domain/chain_sync_state.rs | 92 ++++++++- 8 files changed, 355 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0cd7ffc6cd..5e33e931ee 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3130,6 +3130,7 @@ dependencies = [ "e3-trbfv", "e3-utils", "e3-zk-prover", + "futures", "libp2p", "once_cell", "rayon", diff --git a/crates/ciphernode-builder/Cargo.toml b/crates/ciphernode-builder/Cargo.toml index c6fc85e886..be212b33cc 100644 --- a/crates/ciphernode-builder/Cargo.toml +++ b/crates/ciphernode-builder/Cargo.toml @@ -11,6 +11,7 @@ actix.workspace = true alloy.workspace = true anyhow.workspace = true derivative.workspace = true +futures.workspace = true once_cell.workspace = true e3-aggregator.workspace = true e3-crypto.workspace = true diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index b0733df029..58d2e5efce 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -10,7 +10,7 @@ use crate::{ }; use actix::{Actor, Addr}; use alloy::primitives::Address; -use anyhow::Result; +use anyhow::{ensure, Result}; use derivative::Derivative; use e3_aggregator::ext::{PublicKeyAggregatorExtension, ThresholdPlaintextAggregatorExtension}; use e3_aggregator::CommitteeFinalizer; @@ -23,8 +23,8 @@ use e3_events::{ }; use e3_evm::{ fetch_accusation_vote_validity, fetch_dkg_fold_attestation_verifier, BondingRegistrySolReader, - CiphernodeRegistrySol, CiphernodeRegistrySolReader, InterfoldSolReader, InterfoldSolWriter, - ProviderConfig, SlashingManagerSolReader, SlashingManagerSolWriter, + CiphernodeRegistrySol, CiphernodeRegistrySolReader, EvmChainGatewayHandle, InterfoldSolReader, + InterfoldSolWriter, ProviderConfig, SlashingManagerSolReader, SlashingManagerSolWriter, }; use e3_fhe::ext::FheExtension; use e3_keyshare::ext::ThresholdKeyshareExtension; @@ -73,6 +73,7 @@ pub struct CiphernodeBuilder { in_mem_store: Option>, keyshare: Option, logging: bool, + max_buffered_evm_events: usize, name: Option, multithread_cache: Option>, multithread_concurrent_jobs: Option, @@ -138,6 +139,7 @@ impl CiphernodeBuilder { in_mem_store: None, keyshare: None, logging: false, + max_buffered_evm_events: 100_000, name: None, multithread_cache: None, multithread_concurrent_jobs: None, @@ -377,6 +379,13 @@ impl CiphernodeBuilder { self } + /// Bound decoded EVM events retained per chain while initial sync orders historical and live + /// observations. Exhaustion fails startup instead of silently dropping chain history. + pub fn with_max_buffered_evm_events(mut self, limit: usize) -> Self { + self.max_buffered_evm_events = limit; + self + } + /// Setup a ThresholdPlaintextAggregator pub fn with_threshold_plaintext_aggregation(mut self) -> Self { self.threshold_plaintext_agg = true; @@ -470,6 +479,10 @@ impl CiphernodeBuilder { } pub async fn build(mut self) -> anyhow::Result { + ensure!( + self.max_buffered_evm_events > 0, + "max_buffered_evm_events must be greater than zero" + ); let local_bus = self.resolve_bus(); // Optional event collectors for debugging / testing. @@ -520,7 +533,7 @@ impl CiphernodeBuilder { E3LifecycleCoordinator::attach(&bus, store.clone()).await?; // Setup EVM contract event listeners - let evm_config = self.setup_evm_system(&mut provider_cache, &bus).await?; + let (evm_config, evm_gateways) = self.setup_evm_system(&mut provider_cache, &bus).await?; // Fetch on-chain ZK/slashing configuration let (dkg_fold_verifier_by_chain, accusation_vote_validity_by_chain) = @@ -549,14 +562,17 @@ impl CiphernodeBuilder { setup_net(topic, bus.clone(), eventstore.ts(), interface)?; // Run the sync routine - sync( - &bus, - &evm_config, - &repositories, - &aggregate_config, - &eventstore.seq(), - ) - .await?; + let seq_eventstore = eventstore.seq(); + tokio::try_join!( + sync( + &bus, + &evm_config, + &repositories, + &aggregate_config, + &seq_eventstore, + ), + wait_for_evm_gateways(evm_gateways), + )?; Ok(CiphernodeHandle::new( addr.to_owned(), @@ -635,13 +651,14 @@ impl CiphernodeBuilder { &self, provider_cache: &mut ProviderCache, bus: &BusHandle, - ) -> Result { + ) -> Result<(EvmEventConfig, Vec)> { setup_evm_system( &self.chains, provider_cache, bus, &self.contract_components, self.pubkey_agg, + self.max_buffered_evm_events, ) .await } @@ -918,8 +935,10 @@ async fn setup_evm_system( bus: &BusHandle, contract_components: &ContractComponents, pubkey_agg: bool, -) -> Result { + max_buffered_evm_events: usize, +) -> Result<(EvmEventConfig, Vec)> { let mut evm_config = EvmEventConfig::new(); + let mut gateways = Vec::new(); for chain in chains.iter().filter(|chain| chain.enabled.unwrap_or(true)) { let provider = provider_cache.ensure_read_provider(chain).await?; let chain_id = provider.chain_id(); @@ -930,7 +949,9 @@ async fn setup_evm_system( ProviderConfig::new(rpc_url, chain.rpc_auth.clone()).into_read_provider_factory(); let mut system = EvmSystemChainBuilder::new(bus, &provider); - system.with_provider_factory(provider_factory); + system + .with_provider_factory(provider_factory) + .with_buffer_limit(max_buffered_evm_events); if contract_components.interfold { let write_provider = provider_cache.ensure_write_provider(chain).await?; @@ -1028,8 +1049,18 @@ async fn setup_evm_system( } } - system.build(); + gateways.push(system.build_with_readiness()); } - Ok(evm_config) + Ok((evm_config, gateways)) +} + +async fn wait_for_evm_gateways(gateways: Vec) -> Result<()> { + futures::future::try_join_all( + gateways + .into_iter() + .map(EvmChainGatewayHandle::wait_until_live), + ) + .await?; + Ok(()) } diff --git a/crates/ciphernode-builder/src/evm_system.rs b/crates/ciphernode-builder/src/evm_system.rs index 1ea349cd1d..dbe91a020b 100644 --- a/crates/ciphernode-builder/src/evm_system.rs +++ b/crates/ciphernode-builder/src/evm_system.rs @@ -8,8 +8,8 @@ use actix::Actor; use alloy::{primitives::Address, providers::Provider}; use e3_events::{run_once, BusHandle, EventSubscriber, EventType, HistoricalEvmSyncStart}; use e3_evm::{ - EthProvider, EvmChainGateway, EvmEventProcessor, EvmReadInterface, EvmRouter, Filters, - FixHistoricalOrder, ProviderFactory, + EthProvider, EvmChainGateway, EvmChainGatewayHandle, EvmEventProcessor, EvmReadInterface, + EvmRouter, Filters, FixHistoricalOrder, ProviderFactory, DEFAULT_MAX_BUFFERED_EVM_EVENTS, }; pub trait RouteFn: FnOnce(EvmEventProcessor) -> EvmEventProcessor + Send {} @@ -23,6 +23,7 @@ pub struct EvmSystemChainBuilder

{ provider_factory: Option>, bus: BusHandle, chain_id: u64, + max_buffered_events: usize, route_factories: Vec<(Address, RouteFactory)>, } @@ -34,10 +35,16 @@ impl EvmSystemChainBuilder

{ provider: provider.clone(), provider_factory: None, chain_id, + max_buffered_events: DEFAULT_MAX_BUFFERED_EVM_EVENTS, route_factories: Vec::new(), } } + pub fn with_buffer_limit(&mut self, max_buffered_events: usize) -> &mut Self { + self.max_buffered_events = max_buffered_events; + self + } + pub fn with_provider_factory(&mut self, factory: ProviderFactory

) -> &mut Self { self.provider_factory = Some(factory); self @@ -53,10 +60,16 @@ impl EvmSystemChainBuilder

{ } pub fn build(&mut self) { + drop(self.build_with_readiness()); + } + + pub(crate) fn build_with_readiness(&mut self) -> EvmChainGatewayHandle { // Think about the following in reverse order // Gateway is the final step before connecting to the bus - let next = EvmChainGateway::setup(&self.bus); + let gateway = + EvmChainGateway::setup_with_readiness_and_limit(&self.bus, self.max_buffered_events); + let next = gateway.addr(); // Fix the historical order to avoid missing historical events let next = FixHistoricalOrder::setup(next); @@ -100,6 +113,8 @@ impl EvmSystemChainBuilder

{ // Finaly subscribe to the bus and wait for HistoricalEvmSyncStart self.bus .subscribe(EventType::HistoricalEvmSyncStart, next.recipient()); + + gateway } } diff --git a/crates/config/src/app_config.rs b/crates/config/src/app_config.rs index ebd613c2f3..0f0a626c95 100644 --- a/crates/config/src/app_config.rs +++ b/crates/config/src/app_config.rs @@ -64,6 +64,10 @@ pub struct NodeDefinition { /// protocol operation before this deadline exits non-zero instead of remaining falsely alive. #[serde(default = "default_startup_timeout_secs")] pub startup_timeout_secs: u64, + /// Maximum decoded EVM events retained per chain while initial sync is ordering historical + /// and live data. Exceeding the bound fails startup; events are never silently discarded. + #[serde(default = "default_max_buffered_evm_events")] + pub max_buffered_evm_events: usize, } fn default_multithread_reserve_threads() -> usize { @@ -74,6 +78,10 @@ fn default_startup_timeout_secs() -> u64 { 30 * 60 } +fn default_max_buffered_evm_events() -> usize { + 100_000 +} + impl Default for NodeDefinition { fn default() -> Self { Self { @@ -93,6 +101,7 @@ impl Default for NodeDefinition { multithread_reserve_threads: default_multithread_reserve_threads(), multithread_concurrent_jobs: None, startup_timeout_secs: default_startup_timeout_secs(), + max_buffered_evm_events: default_max_buffered_evm_events(), } } } @@ -180,6 +189,9 @@ impl AppConfig { if node.startup_timeout_secs == 0 { bail!("node.startup_timeout_secs must be greater than zero"); } + if node.max_buffered_evm_events == 0 { + bail!("node.max_buffered_evm_events must be greater than zero"); + } let config_dir_override = (node.config_dir != PathBuf::new()) .then_some(&node.config_dir) @@ -369,6 +381,11 @@ impl AppConfig { pub fn startup_timeout_secs(&self) -> u64 { self.node_def().startup_timeout_secs } + + /// Maximum per-chain decoded-event buffer used before EVM gateways become live. + pub fn max_buffered_evm_events(&self) -> usize { + self.node_def().max_buffered_evm_events + } } #[derive(Debug, Default, Deserialize, Serialize)] @@ -768,6 +785,7 @@ node: r#" node: startup_timeout_secs: 42 + max_buffered_evm_events: 12345 "#, )?; let configured = configured.into_scoped_with_defaults( @@ -777,6 +795,7 @@ node: &PathBuf::from("/my/cwd"), )?; assert_eq!(configured.startup_timeout_secs(), 42); + assert_eq!(configured.max_buffered_evm_events(), 12_345); let default = UnscopedAppConfig::default().into_scoped_with_defaults( "_default", @@ -785,6 +804,7 @@ node: &PathBuf::from("/my/cwd"), )?; assert_eq!(default.startup_timeout_secs(), 30 * 60); + assert_eq!(default.max_buffered_evm_events(), 100_000); Ok(()) } @@ -810,6 +830,28 @@ node: Ok(()) } + #[test] + fn test_zero_evm_buffer_limit_is_rejected() -> Result<()> { + let unscoped: UnscopedAppConfig = serde_yaml::from_str( + r#" +node: + max_buffered_evm_events: 0 +"#, + )?; + let error = unscoped + .into_scoped_with_defaults( + "_default", + &PathBuf::from("/default/data"), + &PathBuf::from("/default/config"), + &PathBuf::from("/my/cwd"), + ) + .expect_err("zero EVM buffer limit must fail configuration"); + assert!(error + .to_string() + .contains("max_buffered_evm_events must be greater than zero")); + Ok(()) + } + #[test] fn test_config_env_vars() { Jail::expect_with(|jail| { diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 979654205c..9a58837a3c 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -57,6 +57,7 @@ pub async fn execute(config: &AppConfig) -> Result { .with_contract_interfold_full() .with_contract_bonding_registry() .with_multithread_config(reserve, concurrent_jobs) + .with_max_buffered_evm_events(config.max_buffered_evm_events()) .with_contract_ciphernode_registry() .with_contract_slashing_manager() .with_trbfv() diff --git a/crates/evm/src/actors/evm_chain_gateway.rs b/crates/evm/src/actors/evm_chain_gateway.rs index ce13c81b8a..f6cf5139f5 100644 --- a/crates/evm/src/actors/evm_chain_gateway.rs +++ b/crates/evm/src/actors/evm_chain_gateway.rs @@ -7,36 +7,90 @@ use crate::domain::chain_sync_state::SyncStatus; use crate::messages::HistoricalSyncComplete; use crate::messages::InterfoldEvmEvent; -use actix::{Actor, Handler}; +use actix::{Actor, ActorContext, Handler}; use actix::{Addr, Recipient}; -use anyhow::Context; -use anyhow::Result; +use anyhow::{bail, Context, Result}; use e3_events::EType; use e3_events::{ - trap, BusHandle, EventSubscriber, EventType, HistoricalEvmEventsReceived, + BusHandle, ErrorDispatcher, EventSubscriber, EventType, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, InterfoldEvent, InterfoldEventData, SyncEnded, Unsequenced, }; use e3_events::{Event, EventPublisher}; use e3_utils::MAILBOX_LIMIT; +use tokio::sync::oneshot; use tracing::warn; +/// Per-chain bound for events accumulated while the node is synchronizing. +/// +/// Tests inject a smaller value. Production deliberately fails startup instead +/// of dropping an observed chain event if this window is exhausted. +pub const DEFAULT_MAX_BUFFERED_EVM_EVENTS: usize = 100_000; + +pub struct EvmChainGatewayHandle { + addr: Addr, + readiness: oneshot::Receiver>, +} + +impl EvmChainGatewayHandle { + pub fn addr(&self) -> Addr { + self.addr.clone() + } + + pub async fn wait_until_live(self) -> Result<()> { + self.readiness + .await + .context("EVM chain gateway stopped before reporting startup status")? + .map_err(anyhow::Error::msg) + } +} + /// This component sits between the Evm ingestion for a chain and the Sync actor and the Bus. /// It coordinates event flow between these components. pub struct EvmChainGateway { bus: BusHandle, status: SyncStatus>, + max_buffered_events: usize, + readiness: Option>>, } impl EvmChainGateway { pub fn new(bus: &BusHandle) -> Self { + Self::with_options(bus, DEFAULT_MAX_BUFFERED_EVM_EVENTS, None) + } + + fn with_options( + bus: &BusHandle, + max_buffered_events: usize, + readiness: Option>>, + ) -> Self { Self { bus: bus.clone(), status: SyncStatus::default(), + max_buffered_events, + readiness, } } pub fn setup(bus: &BusHandle) -> Addr { - let addr = Self::new(bus).start(); + Self::start_and_subscribe(bus, Self::new(bus)) + } + + pub fn setup_with_readiness(bus: &BusHandle) -> EvmChainGatewayHandle { + Self::setup_with_readiness_and_limit(bus, DEFAULT_MAX_BUFFERED_EVM_EVENTS) + } + + pub fn setup_with_readiness_and_limit( + bus: &BusHandle, + max_buffered_events: usize, + ) -> EvmChainGatewayHandle { + let (tx, readiness) = oneshot::channel(); + let actor = Self::with_options(bus, max_buffered_events, Some(tx)); + let addr = Self::start_and_subscribe(bus, actor); + EvmChainGatewayHandle { addr, readiness } + } + + fn start_and_subscribe(bus: &BusHandle, actor: Self) -> Addr { + let addr = actor.start(); bus.subscribe_all( &[EventType::HistoricalEvmSyncStart, EventType::SyncEnded], addr.clone().recipient(), @@ -44,6 +98,24 @@ impl EvmChainGateway { addr } + fn signal_startup(&mut self, result: std::result::Result<(), String>) { + if let Some(sender) = self.readiness.take() { + let _ = sender.send(result); + } + } + + fn fail_closed(&mut self, error: anyhow::Error, ctx: &mut actix::Context) { + let reason = format!( + "EVM chain gateway failed closed: {error:#}. The gateway stopped and will not process \ + further chain events; inspect the snapshot/deploy block and RPC catch-up range, then \ + restart the node to replay chain history" + ); + self.status.fail(reason.clone()); + self.signal_startup(Err(reason.clone())); + self.bus.err(EType::Evm, anyhow::anyhow!(reason)); + ctx.stop(); + } + fn handle_sync_start(&mut self, msg: HistoricalEvmSyncStart) -> Result<()> { let sender = msg .sender @@ -67,6 +139,7 @@ impl EvmChainGateway { for evt in buffer { self.publish_evm_event(evt)?; } + self.signal_startup(Ok(())); Ok(()) } @@ -85,7 +158,12 @@ impl EvmChainGateway { self.process_evm_event(event.into_interfold_event(&self.bus)?)?; Ok(()) } - _ => panic!("EvmChainGateway is only designed to receive InterfoldEvmEvent::HistoricalSyncComplete or InterfoldEvmEvent::Event events"), + InterfoldEvmEvent::Log(_) => { + bail!("EvmChainGateway received an unparsed EVM log") + } + InterfoldEvmEvent::Processed(_) => { + bail!("EvmChainGateway received an internal ordering marker") + } } } @@ -114,13 +192,11 @@ impl EvmChainGateway { } fn process_evm_event(&mut self, msg: InterfoldEvent) -> Result<()> { - match &mut self.status { - SyncStatus::Init { buffer, .. } => buffer.push(msg), - SyncStatus::BufferUntilLive(buffer) => buffer.push(msg), - SyncStatus::ForwardToSyncActor(state) => state.add_event(msg), - SyncStatus::Live => self.publish_evm_event(msg)?, - }; - Ok(()) + if matches!(self.status, SyncStatus::Live) { + return self.publish_evm_event(msg); + } + self.status + .add_buffered_event(msg, self.max_buffered_events) } } @@ -129,26 +205,38 @@ impl Actor for EvmChainGateway { fn started(&mut self, ctx: &mut Self::Context) { ctx.set_mailbox_capacity(MAILBOX_LIMIT) } + + fn stopped(&mut self, _: &mut Self::Context) { + self.signal_startup(Err( + "EVM chain gateway stopped before reaching Live; inspect preceding EVM errors" + .to_owned(), + )); + } } impl Handler for EvmChainGateway { type Result = (); - fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::Evm, &self.bus.with_ec(msg.get_ctx()), || { + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let result = (|| { match msg.into_data() { InterfoldEventData::HistoricalEvmSyncStart(e) => self.handle_sync_start(e)?, InterfoldEventData::SyncEnded(e) => self.handle_sync_ended(e)?, _ => (), } Ok(()) - }) + })(); + if let Err(error) = result { + self.fail_closed(error, ctx); + } } } impl Handler for EvmChainGateway { type Result = (); - fn handle(&mut self, msg: InterfoldEvmEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::Evm, &self.bus.clone(), || self.handle_evm_event(msg)) + fn handle(&mut self, msg: InterfoldEvmEvent, ctx: &mut Self::Context) -> Self::Result { + if let Err(error) = self.handle_evm_event(msg) { + self.fail_closed(error, ctx); + } } } @@ -195,7 +283,8 @@ mod tests { let (tx, mut rx) = mpsc::unbounded_channel(); let collector = SyncEventCollector { tx }.start(); - let addr = EvmChainGateway::setup(&bus); + let gateway = EvmChainGateway::setup_with_readiness(&bus); + let addr = gateway.addr(); let chain_id = 1u64; @@ -243,6 +332,7 @@ mod tests { // The Synchronizer will publish the SyncEnded event when it has all the information it needs // and has published everything to the bus bus.publish_without_context(SyncEnded::new())?; + gateway.wait_until_live().await?; let after_event = EvmEvent::new( CorrelationId::new(), @@ -287,4 +377,53 @@ mod tests { ); Ok(()) } + + #[actix::test] + async fn overflow_emits_actionable_error_stops_and_fails_readiness() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus: BusHandle = system.handle()?.enable("test-overflow"); + let errors = bus.errors(); + let gateway = EvmChainGateway::setup_with_readiness_and_limit(&bus, 1); + let addr = gateway.addr(); + + for entropy in [1, 2] { + let event = EvmEvent::new( + CorrelationId::new(), + TestEvent::new("overflow", entropy).into(), + 100, + u128::from(entropy), + 1, + ); + addr.send(InterfoldEvmEvent::Event(event)).await?; + } + + let startup_error = gateway + .wait_until_live() + .await + .expect_err("overflow must fail gateway readiness") + .to_string(); + assert!(startup_error.contains("Init buffer reached its limit of 1 events")); + assert!(startup_error.contains("will not process further chain events")); + assert!(startup_error.contains("restart the node to replay chain history")); + + let received = errors.send(TakeEvents::new(1)).await?; + assert!(!received.timed_out, "overflow error should be observable"); + let InterfoldEventData::InterfoldError(error) = received.events[0].get_data() else { + panic!("expected an InterfoldError event"); + }; + assert!(error + .message + .contains("Init buffer reached its limit of 1 events")); + assert!(error.message.contains("snapshot/deploy block")); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while addr.connected() { + tokio::task::yield_now().await; + } + }) + .await + .context("overflowed gateway did not stop")?; + assert!(!addr.connected(), "overflowed gateway must stop"); + Ok(()) + } } diff --git a/crates/evm/src/domain/chain_sync_state.rs b/crates/evm/src/domain/chain_sync_state.rs index 5b59e4a81d..163a9b449e 100644 --- a/crates/evm/src/domain/chain_sync_state.rs +++ b/crates/evm/src/domain/chain_sync_state.rs @@ -32,12 +32,6 @@ impl Default for ForwardToSyncActorData { } } -impl ForwardToSyncActorData { - pub(crate) fn add_event(&mut self, event: InterfoldEvent) { - self.buffer.push(event); - } -} - /// State machine coordinating event flow through the `EvmChainGateway`. #[derive(Clone, Debug)] pub(crate) enum SyncStatus { @@ -53,6 +47,8 @@ pub(crate) enum SyncStatus { BufferUntilLive(Vec>), /// Forward all events directly to the bus. Live, + /// A buffer or transition invariant failed. This state is terminal. + Failed { reason: String }, } impl Default for SyncStatus { @@ -65,6 +61,47 @@ impl Default for SyncStatus { } impl SyncStatus { + pub(crate) fn add_buffered_event( + &mut self, + event: InterfoldEvent, + limit: usize, + ) -> Result<()> { + let (state, buffered) = match self { + Self::Init { buffer, .. } => ("Init", buffer.len()), + Self::ForwardToSyncActor(data) => ("ForwardToSyncActor", data.buffer.len()), + Self::BufferUntilLive(buffer) => ("BufferUntilLive", buffer.len()), + Self::Live => bail!("Cannot buffer an EVM event after the gateway is Live"), + Self::Failed { reason } => bail!("EVM chain gateway already failed: {reason}"), + }; + + if buffered >= limit { + let reason = format!( + "EVM chain gateway {state} buffer reached its limit of {limit} events; capacity \ + was exhausted before accepting the next event, so startup is stopping before \ + Live and chain history must be replayed" + ); + *self = Self::Failed { + reason: reason.clone(), + }; + bail!(reason); + } + + match self { + Self::Init { buffer, .. } | Self::BufferUntilLive(buffer) => buffer.push(event), + Self::ForwardToSyncActor(data) => data.buffer.push(event), + Self::Live | Self::Failed { .. } => { + bail!("EVM chain gateway changed state while buffering an event") + } + } + Ok(()) + } + + pub(crate) fn fail(&mut self, reason: impl Into) { + *self = Self::Failed { + reason: reason.into(), + }; + } + pub(crate) fn forward_to_sync_actor( &mut self, sender: S, @@ -118,11 +155,33 @@ impl SyncStatus { #[cfg(test)] mod tests { use super::*; + use e3_events::InterfoldEvent; // A trivial sender stand-in so the state machine can be exercised without actix. #[derive(Clone, Debug, PartialEq)] struct FakeSender(u32); + fn event(id: u64) -> InterfoldEvent { + InterfoldEvent::::test_event("buffer-boundary") + .id(id) + .build() + } + + fn assert_overflow_fails_closed(status: &mut SyncStatus, state: &str) { + status.add_buffered_event(event(1), 2).unwrap(); + status.add_buffered_event(event(2), 2).unwrap(); + + let error = status.add_buffered_event(event(3), 2).unwrap_err(); + let message = error.to_string(); + assert!(message.contains(state), "{message}"); + assert!(message.contains("limit of 2 events"), "{message}"); + assert!(matches!(status, SyncStatus::Failed { .. })); + + let live_error = status.live().unwrap_err().to_string(); + assert!(live_error.contains("Cannot change state to Live")); + assert!(matches!(status, SyncStatus::Failed { .. })); + } + #[test] fn test_happy_path_transitions() { let mut status: SyncStatus = SyncStatus::default(); @@ -159,4 +218,25 @@ mod tests { let (_buffer, returned_pending) = status.forward_to_sync_actor(FakeSender(1)).unwrap(); assert_eq!(returned_pending, Some(pending)); } + + #[test] + fn init_accepts_the_boundary_then_overflow_prevents_live() { + let mut status: SyncStatus = SyncStatus::default(); + assert_overflow_fails_closed(&mut status, "Init"); + } + + #[test] + fn forward_to_sync_accepts_the_boundary_then_overflow_prevents_live() { + let mut status: SyncStatus = SyncStatus::default(); + status.forward_to_sync_actor(FakeSender(1)).unwrap(); + assert_overflow_fails_closed(&mut status, "ForwardToSyncActor"); + } + + #[test] + fn buffer_until_live_accepts_the_boundary_then_overflow_prevents_live() { + let mut status: SyncStatus = SyncStatus::default(); + status.forward_to_sync_actor(FakeSender(1)).unwrap(); + status.buffer_until_live().unwrap(); + assert_overflow_fails_closed(&mut status, "BufferUntilLive"); + } } From 54fc1f286e1613bc4ed520b67218e207f7cea8ed Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 12:06:43 +0500 Subject: [PATCH 15/62] fix(evm): defer live logs until confirmed --- crates/evm/src/actors/evm_read_interface.rs | 41 +++++++++-- crates/evm/src/actors/log_fetcher.rs | 79 +++++++++++++++++++++ 2 files changed, 113 insertions(+), 7 deletions(-) diff --git a/crates/evm/src/actors/evm_read_interface.rs b/crates/evm/src/actors/evm_read_interface.rs index 3a9b26791c..04e15e5a8a 100644 --- a/crates/evm/src/actors/evm_read_interface.rs +++ b/crates/evm/src/actors/evm_read_interface.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::actors::log_fetcher::{ - backfill_to_head, fetch_logs_chunked, process_log, TimestampTracker, + backfill_to_head, fetch_logs_chunked, process_live_log, TimestampTracker, }; use crate::domain::backoff::Backoff; use crate::helpers::{EthProvider, ProviderFactory}; @@ -38,6 +38,9 @@ const PROVIDER_RECREATE_MAX_ATTEMPTS: u32 = 3; const PROVIDER_RECREATE_INITIAL_DELAY_MS: u64 = 2000; /// Consecutive failures before we assume the provider is dead and recreate it. const MAX_RETRIES_BEFORE_RECREATE: u32 = 3; +/// Polling is required even while the subscription is quiet: a log becomes confirmed because later +/// blocks arrive, and those blocks need not contain any matching contract event. +const CONFIRMED_BACKFILL_INTERVAL_SECS: u64 = 5; #[derive(Default, serde::Serialize, serde::Deserialize, Clone)] pub struct EvmReadInterfaceState { @@ -398,6 +401,12 @@ async fn stream_from_evm( consecutive_failures = 0; let sub_id: B256 = *subscription.local_id(); let mut stream = subscription.into_stream(); + let mut confirmation_poll = + tokio::time::interval(Duration::from_secs(CONFIRMED_BACKFILL_INTERVAL_SECS)); + confirmation_poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // Tokio intervals tick immediately once. The outer loop already backfilled, so + // consume that tick and wait for the configured period before querying again. + confirmation_poll.tick().await; info!(chain_id, "Live event subscription active"); loop { @@ -405,12 +414,10 @@ async fn stream_from_evm( maybe_log = stream.next() => { match maybe_log { Some(log) => { - if let Some(bn) = log.block_number { - last_block = last_block.max(bn); - } - process_log( - current_provider.provider(), - log, chain_id, &next, &mut timestamp_tracker, + let _ = process_live_log( + current_provider.provider(), log, chain_id, &next, + &mut timestamp_tracker, &mut last_block, + filters.confirmations(), ).await; } None => { @@ -421,6 +428,26 @@ async fn stream_from_evm( } } } + _ = confirmation_poll.tick(), if filters.confirmations() > 0 => { + if let Err(error) = backfill_to_head( + current_provider.provider(), + &filters.current, + chain_id, + &next, + &mut timestamp_tracker, + &mut last_block, + filters.confirmations(), + ).await { + consecutive_failures += 1; + warn!( + chain_id, + error = %error, + consecutive_failures, + "Confirmed live-log backfill failed; reconnecting" + ); + break; + } + } _ = &mut shutdown => { info!("Shutdown signal received, stopping EVM stream"); let _ = current_provider.provider().unsubscribe(sub_id).await; diff --git a/crates/evm/src/actors/log_fetcher.rs b/crates/evm/src/actors/log_fetcher.rs index 0e929b0b4a..993b7f8864 100644 --- a/crates/evm/src/actors/log_fetcher.rs +++ b/crates/evm/src/actors/log_fetcher.rs @@ -56,6 +56,37 @@ pub(crate) async fn process_log( id } +/// Handle a log delivered by the subscription stream. +/// +/// With a positive confirmation depth the subscription is only a wake-up signal. Publishing the +/// raw notification here would make the historical confirmation gate ineffective, so the periodic +/// canonical backfill owns delivery instead. With zero confirmations this preserves the existing +/// low-latency behavior. +pub(crate) async fn process_live_log( + provider: &L, + log: Log, + chain_id: u64, + next: &EvmEventProcessor, + timestamp_tracker: &mut TimestampTracker, + last_block: &mut u64, + confirmations: u64, +) -> Option { + if confirmations > 0 { + debug!( + chain_id, + block_number = log.block_number, + confirmations, + "Deferring live log to confirmed canonical backfill" + ); + return None; + } + + if let Some(block_number) = log.block_number { + *last_block = (*last_block).max(block_number); + } + Some(process_log(provider, log, chain_id, next, timestamp_tracker).await) +} + /// Fetch logs in chunks from `from_block` to `to_block` with retry logic per chunk. /// Returns the CorrelationId of the last processed event, if any. pub(crate) async fn fetch_logs_chunked( @@ -270,6 +301,10 @@ mod tests { fn get_logs_call_count(&self) -> u32 { self.inner.lock().unwrap().get_logs_calls } + + fn set_block_number(&self, block_number: u64) { + self.inner.lock().unwrap().block_number = block_number; + } } #[async_trait] @@ -516,4 +551,48 @@ mod tests { // Advanced only to the confirmed head, not the raw head of 200. assert_eq!(last_block, 188); } + + #[actix::test] + async fn live_log_waits_for_confirmed_canonical_backfill() -> anyhow::Result<()> { + let mock = MockLogProvider::new(200); + let (next, mut rx) = setup_collector(); + let mut ts = TimestampTracker::new(); + let filter = Filter::new(); + let mut last_block = 188; + + let delivered = process_live_log( + &mock, + make_test_log(200), + 1, + &next, + &mut ts, + &mut last_block, + 12, + ) + .await; + assert!(delivered.is_none()); + assert_eq!(last_block, 188); + tokio::task::yield_now().await; + assert!( + rx.try_recv().is_err(), + "unconfirmed log must not be emitted" + ); + + mock.set_block_number(211); + mock.push_logs(Vec::new()); + backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 12).await?; + assert_eq!(last_block, 199); + tokio::task::yield_now().await; + assert!(rx.try_recv().is_err(), "eleven blocks is not enough"); + + mock.set_block_number(212); + mock.push_logs(vec![make_test_log(200)]); + backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 12).await?; + assert_eq!(last_block, 200); + let emitted = tokio::time::timeout(std::time::Duration::from_secs(1), rx.recv()) + .await? + .expect("confirmed log should be emitted"); + assert!(matches!(emitted, InterfoldEvmEvent::Log(_))); + Ok(()) + } } From e2980300637285b72aa60fc55eacbc2191eb3d77 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 12:20:04 +0500 Subject: [PATCH 16/62] fix(net): bound startup buffering by count and bytes --- .../src/ciphernode_builder.rs | 32 ++- crates/config/src/app_config.rs | 59 ++++++ crates/entrypoint/src/start/start.rs | 4 + crates/net/src/actors/mod.rs | 3 + crates/net/src/actors/net_event_buffer.rs | 185 +++++++++++++++--- crates/net/src/direct_responder.rs | 5 + crates/net/src/domain/net_buffer.rs | 86 ++++++-- crates/net/src/events.rs | 52 +++++ crates/net/src/lib.rs | 34 +++- 9 files changed, 421 insertions(+), 39 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 58d2e5efce..2dd0971295 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -31,7 +31,7 @@ use e3_keyshare::ext::ThresholdKeyshareExtension; use e3_logger::attach_protocol_logger; use e3_multithread::{Multithread, MultithreadReport, TaskPool}; use e3_net::{ - create_channel_bridge, setup_libp2p_keypair, setup_net, setup_net_interface, + create_channel_bridge, setup_libp2p_keypair, setup_net_interface, setup_net_with_limits, NetRepositoryFactory, }; use e3_request::E3LifecycleCoordinator; @@ -74,6 +74,8 @@ pub struct CiphernodeBuilder { keyshare: Option, logging: bool, max_buffered_evm_events: usize, + max_buffered_net_bytes: usize, + max_buffered_net_events: usize, name: Option, multithread_cache: Option>, multithread_concurrent_jobs: Option, @@ -140,6 +142,8 @@ impl CiphernodeBuilder { keyshare: None, logging: false, max_buffered_evm_events: 100_000, + max_buffered_net_bytes: 256 * 1024 * 1024, + max_buffered_net_events: 1_024, name: None, multithread_cache: None, multithread_concurrent_jobs: None, @@ -386,6 +390,14 @@ impl CiphernodeBuilder { self } + /// Bound count and estimated bytes retained from the network while initial synchronization is + /// in progress. Exhaustion fails startup rather than dropping protocol input. + pub fn with_network_buffer_limits(mut self, max_events: usize, max_bytes: usize) -> Self { + self.max_buffered_net_events = max_events; + self.max_buffered_net_bytes = max_bytes; + self + } + /// Setup a ThresholdPlaintextAggregator pub fn with_threshold_plaintext_aggregation(mut self) -> Self { self.threshold_plaintext_agg = true; @@ -483,6 +495,14 @@ impl CiphernodeBuilder { self.max_buffered_evm_events > 0, "max_buffered_evm_events must be greater than zero" ); + ensure!( + self.max_buffered_net_events > 0, + "max_buffered_net_events must be greater than zero" + ); + ensure!( + self.max_buffered_net_bytes > 0, + "max_buffered_net_bytes must be greater than zero" + ); let local_bus = self.resolve_bus(); // Optional event collectors for debugging / testing. @@ -559,7 +579,14 @@ impl CiphernodeBuilder { let topic = "interfold-gossip"; let (peer_id, interface, net_kind) = self.setup_networking(&store, topic).await?; let network_status = interface.status(); - setup_net(topic, bus.clone(), eventstore.ts(), interface)?; + let net_buffer = setup_net_with_limits( + topic, + bus.clone(), + eventstore.ts(), + interface, + self.max_buffered_net_events, + self.max_buffered_net_bytes, + )?; // Run the sync routine let seq_eventstore = eventstore.seq(); @@ -572,6 +599,7 @@ impl CiphernodeBuilder { &seq_eventstore, ), wait_for_evm_gateways(evm_gateways), + net_buffer.wait_until_running(), )?; Ok(CiphernodeHandle::new( diff --git a/crates/config/src/app_config.rs b/crates/config/src/app_config.rs index 0f0a626c95..821be4cd31 100644 --- a/crates/config/src/app_config.rs +++ b/crates/config/src/app_config.rs @@ -68,6 +68,12 @@ pub struct NodeDefinition { /// and live data. Exceeding the bound fails startup; events are never silently discarded. #[serde(default = "default_max_buffered_evm_events")] pub max_buffered_evm_events: usize, + /// Maximum network events retained while startup synchronization is in progress. + #[serde(default = "default_max_buffered_net_events")] + pub max_buffered_net_events: usize, + /// Maximum estimated bytes retained by the network startup buffer. + #[serde(default = "default_max_buffered_net_bytes")] + pub max_buffered_net_bytes: usize, } fn default_multithread_reserve_threads() -> usize { @@ -82,6 +88,14 @@ fn default_max_buffered_evm_events() -> usize { 100_000 } +fn default_max_buffered_net_events() -> usize { + 1_024 +} + +fn default_max_buffered_net_bytes() -> usize { + 256 * 1024 * 1024 +} + impl Default for NodeDefinition { fn default() -> Self { Self { @@ -102,6 +116,8 @@ impl Default for NodeDefinition { multithread_concurrent_jobs: None, startup_timeout_secs: default_startup_timeout_secs(), max_buffered_evm_events: default_max_buffered_evm_events(), + max_buffered_net_events: default_max_buffered_net_events(), + max_buffered_net_bytes: default_max_buffered_net_bytes(), } } } @@ -192,6 +208,12 @@ impl AppConfig { if node.max_buffered_evm_events == 0 { bail!("node.max_buffered_evm_events must be greater than zero"); } + if node.max_buffered_net_events == 0 { + bail!("node.max_buffered_net_events must be greater than zero"); + } + if node.max_buffered_net_bytes == 0 { + bail!("node.max_buffered_net_bytes must be greater than zero"); + } let config_dir_override = (node.config_dir != PathBuf::new()) .then_some(&node.config_dir) @@ -386,6 +408,16 @@ impl AppConfig { pub fn max_buffered_evm_events(&self) -> usize { self.node_def().max_buffered_evm_events } + + /// Maximum count retained by the network startup buffer. + pub fn max_buffered_net_events(&self) -> usize { + self.node_def().max_buffered_net_events + } + + /// Maximum estimated bytes retained by the network startup buffer. + pub fn max_buffered_net_bytes(&self) -> usize { + self.node_def().max_buffered_net_bytes + } } #[derive(Debug, Default, Deserialize, Serialize)] @@ -786,6 +818,8 @@ node: node: startup_timeout_secs: 42 max_buffered_evm_events: 12345 + max_buffered_net_events: 321 + max_buffered_net_bytes: 654321 "#, )?; let configured = configured.into_scoped_with_defaults( @@ -796,6 +830,8 @@ node: )?; assert_eq!(configured.startup_timeout_secs(), 42); assert_eq!(configured.max_buffered_evm_events(), 12_345); + assert_eq!(configured.max_buffered_net_events(), 321); + assert_eq!(configured.max_buffered_net_bytes(), 654_321); let default = UnscopedAppConfig::default().into_scoped_with_defaults( "_default", @@ -805,6 +841,8 @@ node: )?; assert_eq!(default.startup_timeout_secs(), 30 * 60); assert_eq!(default.max_buffered_evm_events(), 100_000); + assert_eq!(default.max_buffered_net_events(), 1_024); + assert_eq!(default.max_buffered_net_bytes(), 256 * 1024 * 1024); Ok(()) } @@ -852,6 +890,27 @@ node: Ok(()) } + #[test] + fn test_zero_network_buffer_limits_are_rejected() -> Result<()> { + for field in ["max_buffered_net_events", "max_buffered_net_bytes"] { + let yaml = format!("node:\n {field}: 0\n"); + let unscoped: UnscopedAppConfig = serde_yaml::from_str(&yaml)?; + let error = unscoped + .into_scoped_with_defaults( + "_default", + &PathBuf::from("/default/data"), + &PathBuf::from("/default/config"), + &PathBuf::from("/my/cwd"), + ) + .expect_err("zero network buffer limit must fail configuration"); + assert!( + error.to_string().contains(field), + "unexpected validation error for {field}: {error}" + ); + } + Ok(()) + } + #[test] fn test_config_env_vars() { Jail::expect_with(|jail| { diff --git a/crates/entrypoint/src/start/start.rs b/crates/entrypoint/src/start/start.rs index 9a58837a3c..00bc486aff 100644 --- a/crates/entrypoint/src/start/start.rs +++ b/crates/entrypoint/src/start/start.rs @@ -58,6 +58,10 @@ pub async fn execute(config: &AppConfig) -> Result { .with_contract_bonding_registry() .with_multithread_config(reserve, concurrent_jobs) .with_max_buffered_evm_events(config.max_buffered_evm_events()) + .with_network_buffer_limits( + config.max_buffered_net_events(), + config.max_buffered_net_bytes(), + ) .with_contract_ciphernode_registry() .with_contract_slashing_manager() .with_trbfv() diff --git a/crates/net/src/actors/mod.rs b/crates/net/src/actors/mod.rs index f3cd62633d..16cfe260bc 100644 --- a/crates/net/src/actors/mod.rs +++ b/crates/net/src/actors/mod.rs @@ -20,6 +20,9 @@ pub use document_publisher::{ handle_document_published_notification, handle_publish_document_requested, DocumentPublisher, EventConverter, }; +pub use net_event_buffer::{ + NetEventBufferHandle, DEFAULT_MAX_BUFFERED_NET_BYTES, DEFAULT_MAX_BUFFERED_NET_EVENTS, +}; pub use net_event_translator::NetEventTranslator; // Internal wiring helpers used by `setup_net`; not part of the public API. diff --git a/crates/net/src/actors/net_event_buffer.rs b/crates/net/src/actors/net_event_buffer.rs index 359c54b07d..c155f2871a 100644 --- a/crates/net/src/actors/net_event_buffer.rs +++ b/crates/net/src/actors/net_event_buffer.rs @@ -4,16 +4,35 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, AsyncContext, Handler, Message}; -use anyhow::{anyhow, Result}; +use actix::{Actor, ActorContext, AsyncContext, Handler, Message}; +use anyhow::{anyhow, Context, Result}; use e3_events::{ - trap, BusHandle, EType, Event, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, + BusHandle, EType, ErrorDispatcher, Event, EventSubscriber, EventType, InterfoldEvent, + InterfoldEventData, }; +use e3_utils::MAILBOX_LIMIT; use tokio::sync::broadcast::{self, error::RecvError}; +use tokio::sync::oneshot; use crate::domain::net_buffer::{BufferDecision, NetEventBufferState}; use crate::events::NetEvent; +pub const DEFAULT_MAX_BUFFERED_NET_EVENTS: usize = 1_024; +pub const DEFAULT_MAX_BUFFERED_NET_BYTES: usize = 256 * 1024 * 1024; + +pub struct NetEventBufferHandle { + readiness: oneshot::Receiver>, +} + +impl NetEventBufferHandle { + pub async fn wait_until_running(self) -> Result<()> { + self.readiness + .await + .context("network event buffer stopped before reporting startup status")? + .map_err(anyhow::Error::msg) + } +} + /// Actor that controls a broadcast channel which will buffer NetEvents until it receives a /// `SyncEnded` event, at which time it releases all buffered events to the output channel. The /// buffering decision logic lives in [`NetEventBufferState`]. @@ -22,21 +41,30 @@ pub struct NetEventBuffer { input_rx: Option>, output_tx: broadcast::Sender, bus: BusHandle, + max_events: usize, + max_bytes: usize, + readiness: Option>>, } impl NetEventBuffer { - pub fn setup( + pub(crate) fn setup_with_limits( bus: &BusHandle, input_rx: &broadcast::Receiver, - ) -> broadcast::Receiver { + max_events: usize, + max_bytes: usize, + ) -> (broadcast::Receiver, NetEventBufferHandle) { let input_rx = input_rx.resubscribe(); - let (output_tx, output_rx) = broadcast::channel(1024); + let (output_tx, output_rx) = broadcast::channel(max_events); + let (readiness_tx, readiness) = oneshot::channel(); let actor = Self { state: NetEventBufferState::syncing(), input_rx: Some(input_rx), - output_tx: output_tx.clone(), + output_tx, bus: bus.clone(), + max_events, + max_bytes, + readiness: Some(readiness_tx), }; let addr = actor.start(); @@ -44,7 +72,7 @@ impl NetEventBuffer { // Subscribe to InterfoldEvent on the bus bus.subscribe(EventType::SyncEnded, addr.clone().recipient()); - output_rx + (output_rx, NetEventBufferHandle { readiness }) } fn handle_interfold_event(&mut self, msg: InterfoldEvent) -> Result<()> { @@ -59,6 +87,7 @@ impl NetEventBuffer { for event in pending { self.forward_event(event)?; } + self.signal_startup(Ok(())); Ok(()) } @@ -68,12 +97,30 @@ impl NetEventBuffer { .map_err(|e| anyhow!("Failed to forward event: {}", e))?; Ok(()) } + + fn signal_startup(&mut self, result: std::result::Result<(), String>) { + if let Some(sender) = self.readiness.take() { + let _ = sender.send(result); + } + } + + fn fail_closed(&mut self, error: anyhow::Error, ctx: &mut actix::Context) { + let reason = format!( + "network event buffer failed closed: {error:#}; startup will stop rather than drop \ + live protocol input. Increase the configured buffer only after measuring the sync \ + backlog, or restore peer/RPC health and restart" + ); + self.signal_startup(Err(reason.clone())); + self.bus.err(EType::Net, anyhow!(reason)); + ctx.stop(); + } } impl Actor for NetEventBuffer { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); // Spawn task to read from broadcast channel let addr = ctx.address(); let mut input_rx = self.input_rx.take().expect("input_rx should be present"); @@ -81,39 +128,79 @@ impl Actor for NetEventBuffer { actix::spawn(async move { loop { match input_rx.recv().await { - Ok(event) => addr.do_send(IncomingNetEvent(event)), - Err(RecvError::Lagged(_)) => continue, + Ok(event) => { + if addr.send(IncomingNetEvent(event)).await.is_err() { + break; + } + } + Err(RecvError::Lagged(skipped)) => { + let _ = addr.send(NetInputLagged(skipped)).await; + break; + } Err(RecvError::Closed) => break, } } }); } + + fn stopped(&mut self, _: &mut Self::Context) { + self.signal_startup(Err( + "network event buffer stopped before startup synchronization completed".to_owned(), + )); + } } #[derive(Message)] #[rtype(result = "()")] struct IncomingNetEvent(NetEvent); +#[derive(Message)] +#[rtype(result = "()")] +struct NetInputLagged(u64); + impl Handler for NetEventBuffer { type Result = (); - fn handle(&mut self, msg: IncomingNetEvent, _: &mut Self::Context) { - trap(EType::Net, &self.bus.clone(), || { - if let BufferDecision::Forward(event) = self.state.observe(msg.0) { - self.forward_event(event)?; - } - Ok(()) - }) + fn handle(&mut self, msg: IncomingNetEvent, ctx: &mut Self::Context) { + let event_bytes = if self.state.is_running() { + 0 + } else { + msg.0.buffered_size_bytes() + }; + let result = self + .state + .observe(msg.0, event_bytes, self.max_events, self.max_bytes) + .and_then(|decision| match decision { + BufferDecision::Buffered => Ok(()), + BufferDecision::Forward(event) => self.forward_event(event), + }); + if let Err(error) = result { + self.fail_closed(error, ctx); + } + } +} + +impl Handler for NetEventBuffer { + type Result = (); + + fn handle(&mut self, msg: NetInputLagged, ctx: &mut Self::Context) { + self.fail_closed( + anyhow!( + "network event input skipped {} events because its bounded broadcast receiver lagged", + msg.0 + ), + ctx, + ); } } impl Handler for NetEventBuffer { type Result = (); - fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::Net, &self.bus.with_ec(msg.get_ctx()), || { - self.handle_interfold_event(msg) - }) + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + if let Err(error) = self.handle_interfold_event(msg) { + self.fail_closed(error, ctx); + } } } @@ -137,7 +224,12 @@ mod tests { let system = EventSystem::new().with_fresh_bus(); let bus = system.handle()?.enable("test"); let (input_tx, input_rx) = broadcast::channel(16); - let mut output_rx = NetEventBuffer::setup(&bus, &input_rx); + let (mut output_rx, handle) = NetEventBuffer::setup_with_limits( + &bus, + &input_rx, + DEFAULT_MAX_BUFFERED_NET_EVENTS, + DEFAULT_MAX_BUFFERED_NET_BYTES, + ); // Send events while syncing - should be buffered let event1 = NetEvent::GossipData(GossipData::GossipBytes(vec![1, 2, 3])); @@ -158,6 +250,7 @@ mod tests { // Send SyncEnded event bus.publish_without_context(SyncEnded::new()).unwrap(); + handle.wait_until_running().await?; // Now buffered events should be forwarded let received1 = output_rx.recv().await.unwrap(); @@ -186,4 +279,52 @@ mod tests { Ok(()) } + + #[actix::test] + async fn startup_buffer_overflow_fails_readiness_without_dropping_oldest() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-overflow"); + let (input_tx, input_rx) = broadcast::channel(16); + let (_output_rx, handle) = + NetEventBuffer::setup_with_limits(&bus, &input_rx, 1, DEFAULT_MAX_BUFFERED_NET_BYTES); + + input_tx.send(NetEvent::GossipData(GossipData::GossipBytes(vec![1])))?; + input_tx.send(NetEvent::GossipData(GossipData::GossipBytes(vec![2])))?; + + let error = timeout(Duration::from_secs(1), handle.wait_until_running()) + .await + .context("network buffer did not report overflow")? + .expect_err("overflow must fail startup readiness") + .to_string(); + assert!(error.contains("events=1/1"), "{error}"); + assert!( + error.contains("startup will stop rather than drop"), + "{error}" + ); + Ok(()) + } + + #[actix::test] + async fn startup_buffer_enforces_estimated_payload_bytes() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-byte-overflow"); + let (input_tx, input_rx) = broadcast::channel(16); + let event = NetEvent::GossipData(GossipData::GossipBytes(vec![0; 32])); + let estimated_bytes = event.buffered_size_bytes(); + let (_output_rx, handle) = + NetEventBuffer::setup_with_limits(&bus, &input_rx, 16, estimated_bytes - 1); + + input_tx.send(event)?; + + let error = timeout(Duration::from_secs(1), handle.wait_until_running()) + .await + .context("network buffer did not report byte overflow")? + .expect_err("byte overflow must fail startup readiness") + .to_string(); + assert!( + error.contains(&format!("next_event_bytes={estimated_bytes}")), + "{error}" + ); + Ok(()) + } } diff --git a/crates/net/src/direct_responder.rs b/crates/net/src/direct_responder.rs index d1c26a2b60..d21bc637ac 100644 --- a/crates/net/src/direct_responder.rs +++ b/crates/net/src/direct_responder.rs @@ -109,6 +109,11 @@ impl DirectResponder { self.request.clone() } + /// Size of the retained request without cloning its payload. + pub fn request_len(&self) -> usize { + self.request.len() + } + /// Get the request data pub fn try_request_into(&self) -> Result where diff --git a/crates/net/src/domain/net_buffer.rs b/crates/net/src/domain/net_buffer.rs index d18a87beb5..c21db8ccd8 100644 --- a/crates/net/src/domain/net_buffer.rs +++ b/crates/net/src/domain/net_buffer.rs @@ -25,33 +25,67 @@ pub(crate) enum BufferDecision { #[derive(Debug)] pub(crate) enum NetEventBufferState { Running, - Syncing(Vec), + Syncing { + events: Vec, + buffered_bytes: usize, + }, + Failed(String), } impl NetEventBufferState { /// Create a new buffer in the syncing state. pub fn syncing() -> Self { - Self::Syncing(Vec::new()) + Self::Syncing { + events: Vec::new(), + buffered_bytes: 0, + } + } + + pub fn is_running(&self) -> bool { + matches!(self, Self::Running) } /// Observe an incoming event, deciding whether to buffer or forward it. - pub fn observe(&mut self, event: NetEvent) -> BufferDecision { + pub fn observe( + &mut self, + event: NetEvent, + event_bytes: usize, + max_events: usize, + max_bytes: usize, + ) -> Result { match self { - Self::Syncing(buffer) => { - buffer.push(event); - BufferDecision::Buffered + Self::Syncing { + events, + buffered_bytes, + } => { + let next_bytes = buffered_bytes.checked_add(event_bytes).ok_or_else(|| { + anyhow::anyhow!("network startup buffer byte accounting overflowed") + })?; + if events.len() >= max_events || next_bytes > max_bytes { + let reason = format!( + "network startup buffer limit exceeded before accepting the next event: \ + events={}/{max_events}, bytes={}/{max_bytes}, next_event_bytes={event_bytes}", + events.len(), buffered_bytes + ); + *self = Self::Failed(reason.clone()); + bail!(reason); + } + events.push(event); + *buffered_bytes = next_bytes; + Ok(BufferDecision::Buffered) } - Self::Running => BufferDecision::Forward(event), + Self::Running => Ok(BufferDecision::Forward(event)), + Self::Failed(reason) => bail!("network startup buffer already failed: {reason}"), } } /// Transition to the running state, returning the events buffered while syncing so the /// caller can flush them. pub fn run(&mut self) -> Result> { - let Self::Syncing(buffer) = self else { - bail!("Cannot change state to Running when state is {:?}", self); + let Self::Syncing { events, .. } = self else { + bail!("Cannot change state to Running when state is {:?}", self) }; - let buffer = std::mem::take(buffer); + let buffer = std::mem::take(events); *self = Self::Running; Ok(buffer) } @@ -69,8 +103,14 @@ mod tests { #[test] fn buffers_events_while_syncing() { let mut state = NetEventBufferState::syncing(); - assert!(matches!(state.observe(event(1)), BufferDecision::Buffered)); - assert!(matches!(state.observe(event(2)), BufferDecision::Buffered)); + assert!(matches!( + state.observe(event(1), 1, 2, 2).unwrap(), + BufferDecision::Buffered + )); + assert!(matches!( + state.observe(event(2), 1, 2, 2).unwrap(), + BufferDecision::Buffered + )); let flushed = state.run().unwrap(); assert_eq!(flushed.len(), 2); } @@ -80,7 +120,7 @@ mod tests { let mut state = NetEventBufferState::syncing(); state.run().unwrap(); assert!(matches!( - state.observe(event(7)), + state.observe(event(7), 1, 1, 1).unwrap(), BufferDecision::Forward(_) )); } @@ -91,4 +131,24 @@ mod tests { state.run().unwrap(); assert!(state.run().is_err()); } + + #[test] + fn event_count_overflow_is_terminal() { + let mut state = NetEventBufferState::syncing(); + state.observe(event(1), 1, 1, 10).unwrap(); + let error = state.observe(event(2), 1, 1, 10).unwrap_err().to_string(); + assert!(error.contains("events=1/1"), "{error}"); + assert!(matches!(state, NetEventBufferState::Failed(_))); + assert!(state.run().is_err()); + } + + #[test] + fn byte_overflow_rejects_event_before_retaining_it() { + let mut state = NetEventBufferState::syncing(); + state.observe(event(1), 4, 10, 5).unwrap(); + let error = state.observe(event(2), 2, 10, 5).unwrap_err().to_string(); + assert!(error.contains("bytes=4/5"), "{error}"); + assert!(error.contains("next_event_bytes=2"), "{error}"); + assert!(matches!(state, NetEventBufferState::Failed(_))); + } } diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index 5164f97cef..b59e2629e2 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -274,6 +274,51 @@ pub enum PutOrStoreError { } impl NetEvent { + /// Conservative size used by the bounded startup buffer. + /// + /// The enum's inline storage is always counted. Heap-backed protocol payloads that can be + /// remotely large are added explicitly; small library error metadata remains covered by the + /// event-count limit. + pub(crate) fn buffered_size_bytes(&self) -> usize { + let dynamic = match self { + Self::GossipData(data) => serialized_size(data), + Self::GossipPublished { message_id, .. } => message_id.0.len(), + Self::DhtGetRecordSucceeded { value, .. } => value.len(), + Self::DhtGetRecordError { error, .. } => match error { + GetRecordError::NotFound { key, closest_peers } => { + key.as_ref().len().saturating_add( + closest_peers + .len() + .saturating_mul(std::mem::size_of::()), + ) + } + GetRecordError::QuorumFailed { key, records, .. } => { + records + .iter() + .fold(key.as_ref().len(), |total, peer_record| { + total + .saturating_add(peer_record.record.key.as_ref().len()) + .saturating_add(peer_record.record.value.len()) + }) + } + GetRecordError::Timeout { key } => key.as_ref().len(), + }, + Self::GossipSubscribed { topic, .. } => topic.as_str().len(), + Self::IncomingRequest(request) => request.responder.request_len(), + Self::OutgoingRequestSucceeded(response) => serialized_size(&response.payload), + Self::OutgoingRequestFailed(response) => response.error.len(), + Self::GossipPublishError { .. } + | Self::DialError { .. } + | Self::ConnectionEstablished { .. } + | Self::OutgoingConnectionError { .. } + | Self::DhtPutRecordSucceeded { .. } + | Self::DhtPutRecordError { .. } + | Self::AllPeersDialed { .. } => 0, + }; + + std::mem::size_of::().saturating_add(dynamic) + } + pub fn correlation_id(&self) -> Option { use NetEvent as N; match self { @@ -290,6 +335,13 @@ impl NetEvent { } } +fn serialized_size(value: &impl Serialize) -> usize { + bincode::serialized_size(value) + .ok() + .and_then(|size| usize::try_from(size).ok()) + .unwrap_or(usize::MAX) +} + /// Payload that is dispatched as a net -> net gossip event from Kademlia. This event signals that /// a document was published and that this node might be interested in it. #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] diff --git a/crates/net/src/lib.rs b/crates/net/src/lib.rs index f8dd13fe7e..6b305e3b88 100644 --- a/crates/net/src/lib.rs +++ b/crates/net/src/lib.rs @@ -79,6 +79,30 @@ pub fn setup_net( eventstore: impl Into>>, interface: impl NetInterface, ) -> Result<()> { + setup_net_with_limits( + topic, + bus, + eventstore, + interface, + DEFAULT_MAX_BUFFERED_NET_EVENTS, + DEFAULT_MAX_BUFFERED_NET_BYTES, + )?; + Ok(()) +} + +/// Set up networking with an explicit fail-closed startup buffer bound and return the readiness +/// handle used by production startup. +pub fn setup_net_with_limits( + topic: &str, + bus: BusHandle, + eventstore: impl Into>>, + interface: impl NetInterface, + max_buffered_events: usize, + max_buffered_bytes: usize, +) -> Result { + if max_buffered_events == 0 || max_buffered_bytes == 0 { + bail!("network startup buffer limits must both be greater than zero"); + } // NOTE: Pass the unbuffered rx to SyncManager as it must operate before live events are // processed let _net_sync = NetSyncManager::setup( @@ -90,7 +114,13 @@ pub fn setup_net( ); // Buffer all incoming events until SyncEnded - let rx = Arc::new(NetEventBuffer::setup(&bus, &interface.rx())); + let (rx, buffer_handle) = NetEventBuffer::setup_with_limits( + &bus, + &interface.rx(), + max_buffered_events, + max_buffered_bytes, + ); + let rx = Arc::new(rx); let tx = interface.tx(); let runner = run_once::({ @@ -107,5 +137,5 @@ pub fn setup_net( bus.subscribe(e3_events::EventType::EffectsEnabled, runner.recipient()); - Ok(()) + Ok(buffer_handle) } From f37451bfb864270dbf903656078e6eed65a1095b Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 12:31:28 +0500 Subject: [PATCH 17/62] fix(shutdown): drain and flush durable state --- Cargo.lock | 1 + crates/ciphernode-builder/src/ciphernode.rs | 72 ++++++++++- crates/ciphernode-builder/src/event_system.rs | 79 +++++++++++- crates/cli/src/start.rs | 31 +++-- crates/data/Cargo.toml | 1 + crates/data/src/commit_log_event_log.rs | 5 + crates/data/src/data_store.rs | 79 +++++++++++- crates/data/src/in_mem.rs | 16 ++- crates/data/src/sled_store.rs | 67 ++++++---- crates/events/src/bus_handle.rs | 122 +++++++++++++++++- crates/events/src/eventbus.rs | 112 +++++++++++++++- crates/events/src/events.rs | 19 +++ crates/events/src/eventstore.rs | 46 ++++++- crates/events/src/eventstore_router.rs | 23 +++- crates/events/src/sequencer.rs | 42 +++++- crates/events/src/snapshot_buffer/batch.rs | 22 +++- .../src/snapshot_buffer/batch_router.rs | 23 +++- crates/events/src/snapshot_buffer/mod.rs | 10 ++ .../src/snapshot_buffer/snapshot_buffer.rs | 25 +++- crates/events/src/traits.rs | 7 + crates/sync/src/actors/sync.rs | 5 +- crates/sync/src/domain/sync_planner.rs | 15 ++- dappnode/docker-compose.yml | 2 + dappnode/tests/test-hardening.sh | 3 + deploy/docker-compose.yml | 12 +- 25 files changed, 762 insertions(+), 77 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e33e931ee..cd3bafa82a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3315,6 +3315,7 @@ dependencies = [ "serde", "sled", "tempfile", + "tokio", "tracing", ] diff --git a/crates/ciphernode-builder/src/ciphernode.rs b/crates/ciphernode-builder/src/ciphernode.rs index 86986b06ab..4cfe87e86d 100644 --- a/crates/ciphernode-builder/src/ciphernode.rs +++ b/crates/ciphernode-builder/src/ciphernode.rs @@ -5,14 +5,30 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use actix::Addr; -use anyhow::Result; +use anyhow::{Context, Result}; use e3_data::{DataStore, InMemStore, StoreAddr}; use e3_events::{BusHandle, HistoryCollector, InterfoldEvent}; use e3_net::{NetChannelBridge, NetworkStatus}; use libp2p::PeerId; +use std::{future::Future, time::Duration}; +use tracing::info; use crate::global_eventstore_cache::EventStoreReader; +async fn enforce_shutdown_deadline(deadline: Duration, shutdown: F) -> Result<()> +where + F: Future>, +{ + tokio::time::timeout(deadline, shutdown) + .await + .with_context(|| { + format!( + "ciphernode shutdown exceeded its {:.3}s deadline", + deadline.as_secs_f64() + ) + })? +} + /// The kind of network interface backing a ciphernode. #[derive(Debug, Clone)] pub enum NetInterfaceKind { @@ -134,4 +150,58 @@ impl CiphernodeHandle { } None } + + /// Stop protocol actors and make persisted state durable within `deadline`. + /// + /// The ordering is deliberate: the persisted `Shutdown` event first stops + /// producers and awaits subscriber final-state handlers; the event pipeline + /// is then flushed; finally snapshot batches drain before the backing store + /// is flushed and closed. + pub async fn shutdown(self, deadline: Duration) -> Result<()> { + enforce_shutdown_deadline(deadline, async move { + info!(stage = "actor-drain", "Ciphernode shutdown barrier started"); + self.bus + .publish_shutdown_and_wait() + .await + .context("failed to persist or acknowledge Shutdown")?; + + info!( + stage = "event-flush", + "Ciphernode shutdown barrier advanced" + ); + self.bus + .flush_event_pipeline() + .await + .context("failed to drain or flush the event pipeline")?; + + info!( + stage = "store-flush", + "Ciphernode shutdown barrier advanced" + ); + self.store + .shutdown() + .await + .context("failed to drain snapshots or flush the data store")?; + Ok(()) + }) + .await + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn shutdown_deadline_is_enforced() { + let error = enforce_shutdown_deadline( + Duration::from_millis(10), + std::future::pending::>(), + ) + .await + .expect_err("pending shutdown must time out") + .to_string(); + + assert!(error.contains("shutdown exceeded its 0.010s deadline")); + } } diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index ec548e26f5..eff69eb193 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -221,8 +221,25 @@ impl EventSystem { pub fn sequencer(&self) -> Result> { self.sequencer .get_or_try_init(|| { - let router = self.eventstore_router()?; - Ok(Sequencer::new(&self.eventbus(), router).start()) + let sequencer = match self.eventstore_addrs()? { + EventStoreAddrs::InMem(addrs) => { + let router = EventStoreRouter::new(addrs).start(); + Sequencer::new_with_flush( + &self.eventbus(), + router.clone().recipient(), + router.recipient(), + ) + } + EventStoreAddrs::Persisted(addrs) => { + let router = EventStoreRouter::new(addrs).start(); + Sequencer::new_with_flush( + &self.eventbus(), + router.clone().recipient(), + router.recipient(), + ) + } + }; + Ok(sequencer.start()) }) .cloned() } @@ -358,14 +375,14 @@ impl EventSystem { let base = b.get_or_init_store(); let buffer = self.buffer()?; buffer.try_send(UpdateDestination::new(base.clone()))?; - DataStore::from_in_mem_with_buffer(&base, self.buffer()?) + DataStore::from_in_mem_with_snapshot_buffer(&base, self.buffer()?) } EventSystemBackend::Persisted(b) => { let base = b.get_or_init_store(&self.handle()?)?; let buffer = self.buffer()?; buffer.try_send(UpdateDestination::new(base))?; - DataStore::from_sled_store_with_buffer( + DataStore::from_sled_store_with_snapshot_buffer( &b.get_or_init_store(&self.handle()?)?, self.buffer()?, ) @@ -415,9 +432,9 @@ mod tests { use tracing::info; use super::*; - use actix::Actor; use actix::Handler; use actix::Message; + use actix::{Actor, ActorContext}; use e3_events::prelude::*; use e3_events::CorrelationId; @@ -444,6 +461,25 @@ mod tests { events: Vec, } + struct ShutdownSnapshotWriter { + store: DataStore, + } + + impl Actor for ShutdownSnapshotWriter { + type Context = actix::Context; + } + + impl Handler for ShutdownSnapshotWriter { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + if matches!(msg.get_data(), InterfoldEventData::Shutdown(_)) { + self.store.write("final-state".to_owned()); + ctx.stop(); + } + } + } + impl Handler for Listener { type Result = (); fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { @@ -496,6 +532,39 @@ mod tests { Ok(()) } + #[actix::test] + async fn shutdown_barrier_persists_final_actor_write_before_store_close() -> Result<()> { + let tmp = TempDir::new()?; + let log_path = tmp.path().join("log"); + let sled_path = tmp.path().join("sled"); + let system = EventSystem::persisted(log_path, sled_path.clone()).with_fresh_bus(); + let bus = system.handle()?.enable("shutdown-test"); + let store = system.store()?; + let final_state = store.base("shutdown/final-state"); + let writer = ShutdownSnapshotWriter { + store: final_state.clone(), + } + .start(); + bus.subscribe(EventType::Shutdown, writer.recipient()); + + bus.publish_shutdown_and_wait().await?; + let late_publish = bus + .publish_without_context(TestEvent::new("late-after-shutdown", 1)) + .expect_err("shutdown must close event admission"); + assert!(late_publish + .to_string() + .contains("event admission is closed")); + bus.flush_event_pipeline().await?; + store.shutdown().await?; + + let disk = e3_data::SledDb::new(&sled_path, "datastore")?; + let bytes = disk + .get(e3_events::Get::new("shutdown/final-state"))? + .expect("final shutdown snapshot must exist"); + assert_eq!(bincode::deserialize::(&bytes)?, "final-state"); + Ok(()) + } + #[actix::test] async fn test_in_mem() { let eventbus = EventBus::::default().start(); diff --git a/crates/cli/src/start.rs b/crates/cli/src/start.rs index ba7349378c..706eecc3fe 100644 --- a/crates/cli/src/start.rs +++ b/crates/cli/src/start.rs @@ -15,11 +15,12 @@ use e3_ciphernode_builder::CiphernodeHandle; use e3_config::AppConfig; use e3_console::Console; use e3_daemon_server::start_daemon_server; -use e3_events::{prelude::*, Shutdown}; use e3_utils::{colorize, Color}; use tokio::signal::unix::{signal, SignalKind}; use tracing::{error, info, instrument}; +const SHUTDOWN_DEADLINE: Duration = Duration::from_secs(30); + #[instrument(skip_all)] pub async fn execute(mut config: AppConfig, peers: Vec) -> Result<()> { // Register signal listeners immediately at startup @@ -41,7 +42,7 @@ pub async fn execute(mut config: AppConfig, peers: Vec) -> Result<()> { result = build_ciphernode(&mut config, peers) => result, // if the shutdown signal completes first then do shutdown without the node _ = &mut shutdown => { - graceful_shutdown(None).await; + graceful_shutdown(None).await?; return Ok(()); } }?; @@ -95,7 +96,7 @@ pub async fn execute(mut config: AppConfig, peers: Vec) -> Result<()> { ); shutdown.await; - graceful_shutdown(Some(node)).await; + graceful_shutdown(Some(node)).await?; Ok(()) } @@ -145,18 +146,26 @@ pub fn shutdown_signal() -> impl std::future::Future { } } -pub async fn graceful_shutdown(node: Option) { +pub async fn graceful_shutdown(node: Option) -> Result<()> { info!("initiating graceful shutdown..."); - if let Some(node) = node { - if let Err(e) = node.bus.publish_without_context(Shutdown) { - error!("Shutdown failed to publish! {e}"); - } - } + let result = match node { + Some(node) => node.shutdown(SHUTDOWN_DEADLINE).await, + None => Ok(()), + }; - tokio::time::sleep(Duration::from_secs(2)).await; - info!("Graceful shutdown complete"); if let Some(logs) = e3_logger::LogCollector::global() { logs.flush(); } + + match result { + Ok(()) => { + info!("Graceful shutdown barrier complete"); + Ok(()) + } + Err(error) => { + error!(%error, "Graceful shutdown failed; process will exit unsuccessfully"); + Err(error) + } + } } diff --git a/crates/data/Cargo.toml b/crates/data/Cargo.toml index f3eed3a413..0092f36118 100644 --- a/crates/data/Cargo.toml +++ b/crates/data/Cargo.toml @@ -17,6 +17,7 @@ serde = { workspace = true } sled = { workspace = true } bincode = { workspace = true } tracing = { workspace = true } +tokio = { workspace = true } async-trait = { workspace = true } once_cell = { workspace = true } tempfile = { workspace = true } diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index c1aa726622..1ad92ccf19 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -107,6 +107,11 @@ impl EventLog for CommitLogEventLog { self.append_bytes(&bytes) } + fn flush(&mut self) -> Result<()> { + self.log.flush().context("Failed to flush event log")?; + Ok(()) + } + fn read_from(&self, from: u64) -> Box)>> { let events = self.read_from_checked(from).unwrap_or_else(|error| { panic!( diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index a9f9921262..a9c6b02c7d 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -7,12 +7,12 @@ use std::borrow::Cow; use crate::{InMemStore, SledStore}; -use actix::{Addr, Recipient}; +use actix::{Addr, Message, Recipient}; use anyhow::anyhow; use anyhow::Context; use anyhow::Result; use e3_events::IntoKey; -use e3_events::{Flush, Get, Insert, InsertSync, Remove}; +use e3_events::{Flush, FlushPendingSnapshots, Get, Insert, InsertSync, Remove, SnapshotBuffer}; use serde::{Deserialize, Serialize}; use tracing::error; @@ -22,6 +22,14 @@ pub enum StoreAddr { Sled(Addr), } +/// Flush and close the underlying store after every snapshot batch has been +/// acknowledged. This message is intentionally separate from the protocol +/// `Shutdown` event so storage remains available while actors persist their +/// final state. +#[derive(Message, Debug)] +#[rtype(result = "Result<()>")] +pub struct ShutdownStore; + impl StoreAddr { pub fn to_maybe_in_mem(&self) -> Option<&Addr> { match self { @@ -42,6 +50,8 @@ pub struct DataStore { insert_sync: Recipient, remove: Recipient, flush: Recipient, + flush_pending_snapshots: Option>, + shutdown: Recipient, } impl DataStore { @@ -86,6 +96,21 @@ impl DataStore { Ok(()) } + /// Drain the snapshot buffer and durably close the backing store. + pub async fn shutdown(&self) -> Result<()> { + if let Some(flush_pending) = &self.flush_pending_snapshots { + flush_pending + .send(FlushPendingSnapshots) + .await + .context("snapshot buffer stopped during shutdown")??; + } + self.shutdown + .send(ShutdownStore) + .await + .context("data store stopped during shutdown")??; + Ok(()) + } + /// Removes data from the scope location pub fn clear(&self) { self.remove.do_send(Remove::new(&self.scope)) @@ -144,6 +169,8 @@ impl DataStore { remove: self.remove.clone(), scope, flush: self.flush.clone(), + flush_pending_snapshots: self.flush_pending_snapshots.clone(), + shutdown: self.shutdown.clone(), } } @@ -156,6 +183,8 @@ impl DataStore { remove: self.remove.clone(), scope: key.into_key(), flush: self.flush.clone(), + flush_pending_snapshots: self.flush_pending_snapshots.clone(), + shutdown: self.shutdown.clone(), } } @@ -172,6 +201,25 @@ impl DataStore { remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), + flush_pending_snapshots: None, + shutdown: addr.clone().recipient(), + } + } + + pub fn from_sled_store_with_snapshot_buffer( + addr: &Addr, + snapshot_buffer: Addr, + ) -> Self { + Self { + addr: StoreAddr::Sled(addr.clone()), + get: addr.clone().recipient(), + insert: snapshot_buffer.clone().recipient(), + insert_sync: addr.clone().recipient(), + remove: addr.clone().recipient(), + scope: vec![], + flush: addr.clone().recipient(), + flush_pending_snapshots: Some(snapshot_buffer.recipient()), + shutdown: addr.clone().recipient(), } } @@ -187,6 +235,25 @@ impl DataStore { remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), + flush_pending_snapshots: None, + shutdown: addr.clone().recipient(), + } + } + + pub fn from_in_mem_with_snapshot_buffer( + addr: &Addr, + snapshot_buffer: Addr, + ) -> Self { + Self { + addr: StoreAddr::InMem(addr.clone()), + get: addr.clone().recipient(), + insert: snapshot_buffer.clone().recipient(), + insert_sync: addr.clone().recipient(), + remove: addr.clone().recipient(), + scope: vec![], + flush: addr.clone().recipient(), + flush_pending_snapshots: Some(snapshot_buffer.recipient()), + shutdown: addr.clone().recipient(), } } @@ -199,6 +266,8 @@ impl DataStore { remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), + flush_pending_snapshots: None, + shutdown: addr.clone().recipient(), } } @@ -211,6 +280,8 @@ impl DataStore { remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), + flush_pending_snapshots: None, + shutdown: addr.clone().recipient(), } } } @@ -225,6 +296,8 @@ impl From<&Addr> for DataStore { remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), + flush_pending_snapshots: None, + shutdown: addr.clone().recipient(), } } } @@ -239,6 +312,8 @@ impl From<&Addr> for DataStore { remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), + flush_pending_snapshots: None, + shutdown: addr.clone().recipient(), } } } diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index bc0ea38c16..ccf846ff59 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -4,8 +4,11 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::in_mem_kv_store::{DataOp, InMemKvStore}; -use actix::{Actor, Handler, Message}; +use crate::{ + in_mem_kv_store::{DataOp, InMemKvStore}, + ShutdownStore, +}; +use actix::{Actor, ActorContext, Handler, Message}; use anyhow::Result; use e3_events::{Flush, Get, Insert, InsertBatch, InsertSync, Remove}; use e3_utils::MAILBOX_LIMIT; @@ -105,6 +108,15 @@ impl Handler for InMemStore { } } +impl Handler for InMemStore { + type Result = Result<()>; + + fn handle(&mut self, _: ShutdownStore, ctx: &mut Self::Context) -> Self::Result { + ctx.stop(); + Ok(()) + } +} + impl Handler for InMemStore { type Result = Vec; fn handle(&mut self, _: GetLog, _: &mut Self::Context) -> Vec { diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 1df72d01d5..5ac4bbaec5 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -4,21 +4,19 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::SledDb; -use actix::{Actor, ActorContext, Addr, Handler}; -use anyhow::Result; -use e3_events::{ - prelude::*, BusHandle, EType, ErrorDispatcher, EventType, Flush, InterfoldEvent, - InterfoldEventData, Unsequenced, -}; +use crate::{ShutdownStore, SledDb}; +use actix::{Actor, ActorContext, Addr, Handler, ResponseFuture}; +use anyhow::{Context, Result}; +use e3_events::{BusHandle, EType, ErrorDispatcher, Flush, InterfoldEvent, Unsequenced}; use e3_events::{Get, Insert, InsertBatch, InsertSync, Remove}; -use e3_utils::{NotifySync, MAILBOX_LIMIT}; +use e3_utils::MAILBOX_LIMIT; use std::path::PathBuf; use tracing::{error, info}; pub struct SledStore { db: Option, bus: Box>>, + write_failure: Option, } impl Actor for SledStore { @@ -30,24 +28,26 @@ impl Actor for SledStore { impl SledStore { pub fn new(bus: &BusHandle, path: &PathBuf) -> Result> { - // Note we pass in a generic BusHandle which supports the err method for passing on errors. - // This was as stores are required before we can initialize the BusHandle to retrieve the - // address so we have a unique node_id. - // If BusHandle is Disabled that is fine as our subscriptions and error publishing function - // remains intact despite it being enabled elsewhere at a later point + // The generic BusHandle is retained only for structured storage errors; + // shutdown itself is coordinated explicitly after actor snapshots drain. info!("Starting SledStore with {:?}", path); let db = SledDb::new(path, "datastore")?; let store = Self { db: Some(db), bus: Box::new(bus.clone()), + write_failure: None, } .start(); - bus.subscribe(EventType::Shutdown, store.clone().into()); - Ok(store) } + + fn record_write_failure(&mut self, error: &anyhow::Error) { + if self.write_failure.is_none() { + self.write_failure = Some(format!("{error:#}")); + } + } } impl Handler for SledStore { @@ -56,6 +56,7 @@ impl Handler for SledStore { fn handle(&mut self, event: Insert, _: &mut Self::Context) -> Self::Result { if let Some(ref mut db) = &mut self.db { if let Err(err) = db.insert(event) { + self.record_write_failure(&err); self.bus.err(EType::Data, err) } } @@ -68,6 +69,7 @@ impl Handler for SledStore { fn handle(&mut self, event: InsertBatch, _: &mut Self::Context) -> Self::Result { if let Some(ref mut db) = &mut self.db { if let Err(err) = db.insert_batch(event.commands()) { + self.record_write_failure(&err); self.bus.err(EType::Data, err) } } @@ -79,8 +81,10 @@ impl Handler for SledStore { fn handle(&mut self, event: InsertSync, _: &mut Self::Context) -> Self::Result { if let Some(ref mut db) = &mut self.db { - db.insert(event.into()) - .map_err(|e| anyhow::anyhow!("{}", e.to_string()))? + if let Err(error) = db.insert(event.into()) { + self.record_write_failure(&error); + return Err(error); + } } Ok(()) } @@ -92,6 +96,7 @@ impl Handler for SledStore { fn handle(&mut self, event: Remove, _: &mut Self::Context) -> Self::Result { if let Some(ref mut db) = &mut self.db { if let Err(err) = db.remove(event) { + self.record_write_failure(&err); self.bus.err(EType::Data, err) } } @@ -122,19 +127,31 @@ impl Handler for SledStore { fn handle(&mut self, _: Flush, _: &mut Self::Context) -> Self::Result { if let Some(ref db) = self.db { if let Err(err) = db.flush() { + self.record_write_failure(&err); self.bus.err(EType::Data, err) } } } } -impl Handler for SledStore { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - if let InterfoldEventData::Shutdown(_) = msg.get_data() { - self.notify_sync(ctx, Flush); // Flush all pending writes - let _db = self.db.take(); // db will be dropped - ctx.stop() - } +impl Handler for SledStore { + type Result = ResponseFuture>; + + fn handle(&mut self, _: ShutdownStore, ctx: &mut Self::Context) -> Self::Result { + let db = self.db.take(); + let write_failure = self.write_failure.take(); + ctx.stop(); + + Box::pin(async move { + let db = db.context("SledStore was already closed")?; + tokio::task::spawn_blocking(move || db.flush()) + .await + .context("SledStore flush task failed")??; + + if let Some(error) = write_failure { + anyhow::bail!("SledStore observed a write failure before shutdown: {error}"); + } + Ok(()) + }) } } diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index b8064b7fc1..860fc242cd 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -8,8 +8,17 @@ use actix::{Actor, Addr, Handler, Recipient}; use anyhow::Result; use derivative::Derivative; use e3_utils::{actix::channel::oneshot, MAILBOX_LIMIT}; -use std::{future::Future, marker::PhantomData, pin::Pin}; -use tracing::error; +use std::{ + future::Future, + marker::PhantomData, + pin::Pin, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }, +}; +use tokio::sync::Notify; +use tracing::{error, warn}; use crate::{ event_context::EventContext, @@ -20,8 +29,9 @@ use crate::{ ErrorDispatcher, ErrorFactory, EventConstructorWithTimestamp, EventContextAccessors, EventFactory, EventPublisher, EventSubscriber, }, - EType, ErrorEvent, EventBus, EventContextManager, EventSource, EventType, HistoryCollector, - InterfoldEvent, InterfoldEventData, Sequenced, Subscribe, Unsequenced, Unsubscribe, + EType, ErrorEvent, EventBus, EventBusBarrier, EventContextManager, EventSource, EventType, + FlushEventStores, HistoryCollector, InterfoldEvent, InterfoldEventData, Sequenced, + SequencerBarrier, Shutdown, Subscribe, Unsequenced, Unsubscribe, }; /// Typestate marker indicating the BusHandle has not yet been enabled with an HLC clock. @@ -32,6 +42,58 @@ pub struct Disabled; #[derive(Clone, Debug, PartialEq, Eq)] pub struct Enabled; +#[derive(Debug)] +struct EventAdmission { + accepting: AtomicBool, + in_flight: AtomicUsize, + drained: Notify, +} + +impl EventAdmission { + fn new() -> Self { + Self { + accepting: AtomicBool::new(true), + in_flight: AtomicUsize::new(0), + drained: Notify::new(), + } + } + + fn enter(self: &Arc) -> Result { + if !self.accepting.load(Ordering::Acquire) { + anyhow::bail!("event admission is closed because node shutdown has started"); + } + + self.in_flight.fetch_add(1, Ordering::AcqRel); + let guard = EventAdmissionGuard(Arc::clone(self)); + if !self.accepting.load(Ordering::Acquire) { + drop(guard); + anyhow::bail!("event admission is closed because node shutdown has started"); + } + Ok(guard) + } + + async fn close_and_wait(&self) { + self.accepting.store(false, Ordering::Release); + loop { + let drained = self.drained.notified(); + if self.in_flight.load(Ordering::Acquire) == 0 { + return; + } + drained.await; + } + } +} + +struct EventAdmissionGuard(Arc); + +impl Drop for EventAdmissionGuard { + fn drop(&mut self) { + if self.0.in_flight.fetch_sub(1, Ordering::AcqRel) == 1 { + self.0.drained.notify_one(); + } + } +} + #[derive(Derivative)] #[derivative( Clone(bound = ""), @@ -49,6 +111,9 @@ pub struct BusHandle { hlc: HlcFactory, /// Temporary context for events the bus publishes ctx: Option>, + /// Shared admission fence closed before the shutdown event is enqueued. + #[derivative(Debug = "ignore", PartialEq = "ignore")] + admission: Arc, #[derivative(Debug = "ignore", PartialEq = "ignore")] _state: PhantomData, } @@ -65,6 +130,7 @@ impl BusHandle { sequencer, hlc, ctx: None, + admission: Arc::new(EventAdmission::new()), _state: PhantomData, } } @@ -78,6 +144,7 @@ impl BusHandle { sequencer: self.sequencer, hlc: self.hlc, ctx: None, + admission: self.admission, _state: PhantomData, } } @@ -90,6 +157,7 @@ impl BusHandle { sequencer: self.sequencer, hlc: self.hlc, ctx: None, + admission: self.admission, _state: PhantomData, } } @@ -182,15 +250,55 @@ impl EventPublisher> for BusHandle { } fn naked_dispatch(&self, event: InterfoldEvent) { + let Ok(_admission) = self.admission.enter() else { + warn!("Dropping an internal event because node shutdown has closed admission"); + return; + }; self.sequencer.do_send(event); } } impl BusHandle { pub async fn naked_dispatch_async(&self, event: InterfoldEvent) -> Result<()> { + let _admission = self.admission.enter()?; self.sequencer.send(event).await?; Ok(()) } + + /// Persist and broadcast `Shutdown`, waiting until every live EventBus + /// subscriber has completed its shutdown handler. + pub async fn publish_shutdown_and_wait(&self) -> Result> { + let (recipient, response) = oneshot::>(); + self.event_bus + .send(Subscribe::new(EventType::Shutdown, recipient.clone())) + .await?; + + // Refuse new work and wait until every publisher that entered before + // the fence has enqueued its event. Shutdown is therefore ordered after + // all admitted work and before every rejected late arrival. + self.admission.close_and_wait().await; + let event = self.event_from(Shutdown, None)?; + self.sequencer.send(event).await?; + + let shutdown = response.await?; + // Awaiting this mailbox operation also proves the EventBus has + // completed its paused, acknowledged Shutdown fanout. + self.event_bus + .send(Unsubscribe::new(EventType::Shutdown, recipient)) + .await?; + Ok(shutdown) + } + + /// Drain and flush the event pipeline after producers have observed + /// `Shutdown`: sequencer -> event-store router -> logs -> sequencer + /// responses -> EventBus fanout. + pub async fn flush_event_pipeline(&self) -> Result<()> { + self.sequencer.send(FlushEventStores).await??; + self.sequencer.send(SequencerBarrier).await?; + self.event_bus.send(EventBusBarrier).await?; + Ok(()) + } + fn publish_from_remote_impl( &self, data: impl Into, @@ -199,6 +307,7 @@ impl BusHandle { block: Option, source: EventSource, ) -> Result<()> { + let _admission = self.admission.enter()?; let evt = self.event_from_remote_source(data, caused_by, remote_ts, block, source)?; self.sequencer.do_send(evt); Ok(()) @@ -208,6 +317,7 @@ impl BusHandle { data: impl Into, caused_by: Option>, ) -> Result<()> { + let _admission = self.admission.enter()?; let evt = self.event_from(data, caused_by)?; self.sequencer.do_send(evt); Ok(()) @@ -216,6 +326,10 @@ impl BusHandle { impl ErrorDispatcher> for BusHandle { fn err(&self, err_type: EType, error: anyhow::Error) { + let Ok(_admission) = self.admission.enter() else { + error!(%error, "Error event not admitted because node shutdown has started"); + return; + }; match self.event_from_error(err_type, error, self.get_ctx()) { Ok(evt) => self.sequencer.do_send(evt), Err(e) => error!("{e}"), diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index bdd6ce71d1..a1b3443c51 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::traits::{ErrorEvent, Event}; -use crate::EventType; +use crate::{EventBusBarrier, EventType}; use actix::prelude::*; use e3_utils::{colorize, Color, MAILBOX_LIMIT, MAILBOX_LIMIT_LARGE}; use std::collections::{HashMap, HashSet, VecDeque}; @@ -177,17 +177,49 @@ impl Default for EventBus { impl Handler for EventBus { type Result = (); - fn handle(&mut self, event: E, _: &mut Context) { + fn handle(&mut self, event: E, ctx: &mut Context) { if self.is_duplicate(&event) { return; } + + let event_type = event.event_type(); + if event_type == EventType::Shutdown.as_str() { + let listeners = self + .listeners + .get("*") + .into_iter() + .flatten() + .chain(self.listeners.get(&event_type).into_iter().flatten()) + .cloned() + .collect::>(); + + tracing::info!( + listeners = listeners.len(), + "Acknowledging Shutdown across EventBus subscribers" + ); + tracing::info!("{} {}", colorize(">>>", Color::Yellow), event); + self.track(&event); + + ctx.wait( + async move { + for listener in listeners { + if let Err(error) = listener.send(event.clone()).await { + tracing::warn!(%error, "Shutdown subscriber was already unavailable"); + } + } + } + .into_actor(self), + ); + return; + } + if let Some(listeners) = self.listeners.get("*") { for listener in listeners { listener.do_send(event.clone()); } } - if let Some(listeners) = self.listeners.get(&event.event_type()) { + if let Some(listeners) = self.listeners.get(&event_type) { for listener in listeners { listener.do_send(event.clone()); } @@ -198,6 +230,12 @@ impl Handler for EventBus { } } +impl Handler for EventBus { + type Result = (); + + fn handle(&mut self, _: EventBusBarrier, _: &mut Context) {} +} + ////////////////////////////////////////////////////////////////////////////// // Subscribe Message ////////////////////////////////////////////////////////////////////////////// @@ -485,6 +523,11 @@ mod tests { use super::*; use crate::{AggregateId, WithAggregateId}; use std::hash::{Hash, Hasher}; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; + use tokio::sync::Notify; #[derive(Clone, Debug, Eq, PartialEq)] struct CollidingId(u64); @@ -517,6 +560,7 @@ mod tests { struct CollidingEvent { id: CollidingId, data: TestData, + event_type: &'static str, } impl CollidingEvent { @@ -524,6 +568,15 @@ mod tests { Self { id: CollidingId(id), data: TestData, + event_type: "CollidingEvent", + } + } + + fn shutdown(id: u64) -> Self { + Self { + id: CollidingId(id), + data: TestData, + event_type: EventType::Shutdown.as_str(), } } } @@ -543,7 +596,7 @@ mod tests { } fn event_type(&self) -> String { - "CollidingEvent".to_owned() + self.event_type.to_owned() } fn get_data(&self) -> &Self::Data { @@ -555,6 +608,28 @@ mod tests { } } + struct BlockingShutdownSubscriber { + gate: Arc, + completed: Arc, + } + + impl Actor for BlockingShutdownSubscriber { + type Context = actix::Context; + } + + impl Handler for BlockingShutdownSubscriber { + type Result = ResponseFuture<()>; + + fn handle(&mut self, _: CollidingEvent, _: &mut Self::Context) -> Self::Result { + let gate = Arc::clone(&self.gate); + let completed = Arc::clone(&self.completed); + Box::pin(async move { + gate.notified().await; + completed.store(true, Ordering::SeqCst); + }) + } + } + #[actix::test] async fn distinct_ids_with_identical_hashes_are_both_delivered() -> anyhow::Result<()> { let bus = EventBus::with_dedup_capacity(EventBusConfig::default(), 4).start(); @@ -616,4 +691,33 @@ mod tests { assert_eq!(received.events.len(), 2); Ok(()) } + + #[actix::test] + async fn shutdown_pauses_event_bus_until_subscriber_acknowledges() -> anyhow::Result<()> { + let bus = EventBus::with_dedup_capacity(EventBusConfig::default(), 2).start(); + let gate = Arc::new(Notify::new()); + let completed = Arc::new(AtomicBool::new(false)); + let subscriber = BlockingShutdownSubscriber { + gate: Arc::clone(&gate), + completed: Arc::clone(&completed), + } + .start(); + + bus.send(Subscribe::new(EventType::Shutdown, subscriber.recipient())) + .await?; + bus.send(CollidingEvent::shutdown(9)).await?; + + let mut fence = Box::pin(bus.send(EventBusBarrier)); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut fence) + .await + .is_err(), + "EventBus fence completed before the shutdown subscriber" + ); + + gate.notify_one(); + fence.await?; + assert!(completed.load(Ordering::SeqCst)); + Ok(()) + } } diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index 1969227406..c011a351ff 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; use actix::{Message, Recipient}; +use anyhow::Result; use crate::{AggregateId, CorrelationId, EventSource, InterfoldEvent, Sequenced, Unsequenced}; @@ -68,6 +69,24 @@ impl StoreEventResponse { } } +/// Flush every event store after all previously routed appends have completed. +/// Used by the clean-shutdown barrier; failures must reach the caller. +#[derive(Message, Debug)] +#[rtype(result = "Result<()>")] +pub struct FlushEventStores; + +/// A no-op sequencer mailbox fence. Once its response arrives, every earlier +/// store response has been forwarded to the EventBus. +#[derive(Message, Debug)] +#[rtype(result = "()")] +pub struct SequencerBarrier; + +/// A no-op EventBus mailbox fence. Once handled, every earlier event has been +/// processed by every live downstream subscriber. +#[derive(Message, Debug)] +#[rtype(result = "()")] +pub struct EventBusBarrier; + /// Trait for various EventStore query types pub trait QueryKind { type Shape: Send; diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 01e9694775..8e65b03ca3 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - events::{StoreEventRequested, StoreEventResponse}, + events::{FlushEventStores, StoreEventRequested, StoreEventResponse}, Event, EventContextAccessors, EventLog, EventStoreFilter, EventStoreQueryBy, EventStoreQueryResponse, InterfoldEvent, Seq, SequenceIndex, Sequenced, Ts, Unsequenced, }; @@ -14,6 +14,7 @@ use anyhow::{bail, Result}; use tracing::{error, warn}; const INDEX_RECONCILE_PAGE_SIZE: usize = 1_024; + pub struct EventStore { index: I, log: L, @@ -253,6 +254,14 @@ impl Handler for EventStore< } } +impl Handler for EventStore { + type Result = Result<()>; + + fn handle(&mut self, _: FlushEventStores, _: &mut Self::Context) -> Self::Result { + self.log.flush() + } +} + impl Handler> for EventStore { type Result = (); fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { @@ -334,6 +343,7 @@ mod tests { events: Vec>, bounded_read_calls: Option>, bounded_read_limit: Option>, + flushes: Option>, unbounded_read_calls: Option>, } @@ -343,6 +353,7 @@ mod tests { events: Vec::new(), bounded_read_calls: None, bounded_read_limit: None, + flushes: None, unbounded_read_calls: None, } } @@ -352,6 +363,17 @@ mod tests { events: Vec::new(), bounded_read_calls: None, bounded_read_limit: Some(tracker), + flushes: None, + unbounded_read_calls: None, + } + } + + fn with_flush_tracker(tracker: Arc) -> Self { + Self { + events: Vec::new(), + bounded_read_calls: None, + bounded_read_limit: None, + flushes: Some(tracker), unbounded_read_calls: None, } } @@ -366,6 +388,7 @@ mod tests { events, bounded_read_calls: Some(bounded_read_calls), bounded_read_limit: Some(bounded_read_limit), + flushes: None, unbounded_read_calls: Some(unbounded_read_calls), } } @@ -377,6 +400,13 @@ mod tests { Ok(self.events.len() as u64) } + fn flush(&mut self) -> Result<()> { + if let Some(flushes) = &self.flushes { + flushes.fetch_add(1, Ordering::SeqCst); + } + Ok(()) + } + fn read_from( &self, from: u64, @@ -491,6 +521,20 @@ mod tests { assert_eq!(logged.len(), 2); } + #[actix::test] + async fn shutdown_flush_message_reaches_event_log() -> Result<()> { + let flushes = Arc::new(AtomicUsize::new(0)); + let store = EventStore::new( + MockIndex::new(), + MockLog::with_flush_tracker(Arc::clone(&flushes)), + ) + .start(); + + store.send(FlushEventStores).await??; + + assert_eq!(flushes.load(Ordering::SeqCst), 1); + Ok(()) + } #[test] fn startup_index_reconciliation_reads_log_in_bounded_pages() { let event_count = INDEX_RECONCILE_PAGE_SIZE * 2 + 2; diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index 7816a0cc7e..a0ba826e40 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -5,12 +5,12 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::eventstore::EventStore; use crate::{ - events::{EventStoreQueryResponse, StoreEventRequested}, + events::{EventStoreQueryResponse, FlushEventStores, StoreEventRequested}, AggregateId, EventContextAccessors, EventLog, SequenceIndex, }; use crate::{CorrelationId, Die, EventStoreQueryBy, InterfoldEvent, Seq, SeqAgg, Ts, TsAgg}; -use actix::{Actor, ActorContext, Addr, AsyncContext, Context, Handler, Recipient}; -use anyhow::Result; +use actix::{Actor, ActorContext, Addr, AsyncContext, Context, Handler, Recipient, ResponseFuture}; +use anyhow::{Context as _, Result}; use e3_utils::MAILBOX_LIMIT_LARGE; use std::collections::HashMap; use tracing::{debug, error, warn}; @@ -222,6 +222,23 @@ impl Handler for EventStoreR } } +impl Handler for EventStoreRouter { + type Result = ResponseFuture>; + + fn handle(&mut self, _: FlushEventStores, _: &mut Self::Context) -> Self::Result { + let stores: Vec<_> = self.stores.values().cloned().collect(); + Box::pin(async move { + for store in stores { + store + .send(FlushEventStores) + .await + .context("event store stopped before its shutdown flush")??; + } + Ok(()) + }) + } +} + impl Handler> for EventStoreRouter { type Result = (); diff --git a/crates/events/src/sequencer.rs b/crates/events/src/sequencer.rs index 3c383c117b..026617df49 100644 --- a/crates/events/src/sequencer.rs +++ b/crates/events/src/sequencer.rs @@ -5,16 +5,18 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use crate::{ - events::{StoreEventRequested, StoreEventResponse}, + events::{FlushEventStores, SequencerBarrier, StoreEventRequested, StoreEventResponse}, EventBus, InterfoldEvent, Sequenced, Unsequenced, }; -use actix::{Actor, Addr, AsyncContext, Handler, Recipient}; +use actix::{Actor, Addr, AsyncContext, Handler, Recipient, ResponseFuture}; +use anyhow::{Context, Result}; use e3_utils::MAILBOX_LIMIT; /// Component to sequence the storage of events pub struct Sequencer { bus: Addr>>, eventstore: Recipient, + eventstore_flush: Option>, } impl Sequencer { @@ -25,6 +27,19 @@ impl Sequencer { Self { bus: bus.clone(), eventstore: eventstore.into(), + eventstore_flush: None, + } + } + + pub fn new_with_flush( + bus: &Addr>>, + eventstore: impl Into>, + eventstore_flush: impl Into>, + ) -> Self { + Self { + bus: bus.clone(), + eventstore: eventstore.into(), + eventstore_flush: Some(eventstore_flush.into()), } } @@ -60,6 +75,29 @@ impl Handler for Sequencer { } } +impl Handler for Sequencer { + type Result = ResponseFuture>; + + fn handle(&mut self, _: FlushEventStores, _: &mut Self::Context) -> Self::Result { + let eventstore_flush = self.eventstore_flush.clone(); + Box::pin(async move { + let eventstore_flush = eventstore_flush + .context("sequencer was constructed without an event-store flush endpoint")?; + eventstore_flush + .send(FlushEventStores) + .await + .context("event-store router stopped during shutdown flush")??; + Ok(()) + }) + } +} + +impl Handler for Sequencer { + type Result = (); + + fn handle(&mut self, _: SequencerBarrier, _: &mut Self::Context) -> Self::Result {} +} + #[cfg(test)] mod tests { use e3_ciphernode_builder::EventSystem; diff --git a/crates/events/src/snapshot_buffer/batch.rs b/crates/events/src/snapshot_buffer/batch.rs index e6ca43a1ea..e0f40a090d 100644 --- a/crates/events/src/snapshot_buffer/batch.rs +++ b/crates/events/src/snapshot_buffer/batch.rs @@ -4,7 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use actix::{Actor, ActorContext, Addr, AsyncContext, Handler, Message, Recipient}; +use super::FlushPendingSnapshots; +use actix::{Actor, ActorContext, Addr, AsyncContext, Handler, Message, Recipient, ResponseFuture}; +use anyhow::{Context, Result}; use e3_utils::MAILBOX_LIMIT; use crate::{Die, Insert, InsertBatch}; @@ -55,6 +57,24 @@ impl Handler for Batch { } } +impl Handler for Batch { + type Result = ResponseFuture>; + + fn handle(&mut self, _: FlushPendingSnapshots, ctx: &mut Self::Context) -> Self::Result { + let inserts = std::mem::take(&mut self.inserts); + let db = self.db.clone(); + ctx.stop(); + Box::pin(async move { + if !inserts.is_empty() { + db.send(InsertBatch::new(inserts)) + .await + .context("snapshot destination stopped before accepting its final batch")?; + } + Ok(()) + }) + } +} + impl Handler for Batch { type Result = (); fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { diff --git a/crates/events/src/snapshot_buffer/batch_router.rs b/crates/events/src/snapshot_buffer/batch_router.rs index 9596832da7..53df14a0e2 100644 --- a/crates/events/src/snapshot_buffer/batch_router.rs +++ b/crates/events/src/snapshot_buffer/batch_router.rs @@ -6,13 +6,14 @@ use super::{ batch::{Batch, Flush}, timelock_queue::{Clock, StartTimelock}, - AggregateConfig, UpdateDestination, + AggregateConfig, FlushPendingSnapshots, UpdateDestination, }; use crate::{ AggregateId, EventContextAccessors, EventContextSeq, EventType, Insert, InsertBatch, InterfoldEvent, Sequenced, StoreKeys, }; -use actix::{Actor, Addr, Handler, Message, Recipient}; +use actix::{Actor, Addr, Handler, Message, Recipient, ResponseFuture}; +use anyhow::{Context, Result}; use e3_utils::MAILBOX_LIMIT; use std::{collections::HashMap, sync::Arc, time::Duration}; use tracing::{debug, error}; @@ -210,6 +211,24 @@ impl Handler for BatchRouter { } } +impl Handler for BatchRouter { + type Result = ResponseFuture>; + + fn handle(&mut self, _: FlushPendingSnapshots, _: &mut Self::Context) -> Self::Result { + let batches: Vec<_> = self.batches.drain().map(|(_, batch)| batch).collect(); + self.aggregates.clear(); + Box::pin(async move { + for batch in batches { + batch + .send(FlushPendingSnapshots) + .await + .context("snapshot batch stopped before its final flush")??; + } + Ok(()) + }) + } +} + impl Handler for BatchRouter { type Result = (); fn handle(&mut self, msg: UpdateDestination, _: &mut Self::Context) -> Self::Result { diff --git a/crates/events/src/snapshot_buffer/mod.rs b/crates/events/src/snapshot_buffer/mod.rs index 44619750a0..73a2353d7a 100644 --- a/crates/events/src/snapshot_buffer/mod.rs +++ b/crates/events/src/snapshot_buffer/mod.rs @@ -10,6 +10,16 @@ mod batch_router; mod snapshot_buffer; mod timelock_queue; +use actix::Message; +use anyhow::Result; + +/// Acknowledged shutdown fence that drains every open snapshot batch into its +/// destination store. Unlike the normal debounce flush, completion means all +/// previously queued `InsertBatch` handlers have run. +#[derive(Message, Debug)] +#[rtype(result = "Result<()>")] +pub struct FlushPendingSnapshots; + pub use aggregate_config::*; pub use snapshot_buffer::*; pub use timelock_queue::Tick; diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index 56ebb69011..40bba6eba0 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -6,11 +6,11 @@ use super::{ batch_router::{BatchRouter, FlushSeq}, timelock_queue::{Clock, StartTimelock, SystemClock, Tick, TimelockQueue}, - AggregateConfig, + AggregateConfig, FlushPendingSnapshots, }; use crate::{Insert, InsertBatch, InterfoldEvent}; -use actix::{Actor, Addr, Handler, Message, Recipient}; -use anyhow::Result; +use actix::{Actor, Addr, Handler, Message, Recipient, ResponseFuture}; +use anyhow::{Context, Result}; use e3_utils::MAILBOX_LIMIT; use std::sync::Arc; use tracing::{info, trace}; @@ -141,6 +141,22 @@ impl Handler for SnapshotBuffer { } } +impl Handler for SnapshotBuffer { + type Result = ResponseFuture>; + + fn handle(&mut self, _: FlushPendingSnapshots, _: &mut Self::Context) -> Self::Result { + let router = self.router.clone(); + Box::pin(async move { + let router = router.context("snapshot buffer has no batch router")?; + router + .send(FlushPendingSnapshots) + .await + .context("snapshot batch router stopped during final flush")??; + Ok(()) + }) + } +} + impl Handler for SnapshotBuffer { type Result = (); fn handle(&mut self, msg: Tick, _: &mut Self::Context) -> Self::Result { @@ -197,7 +213,7 @@ mod mock_store { mod tests { use super::super::timelock_queue::mock_clock::MockClock; use super::mock_store::GetEvts; - use super::{mock_store, SnapshotBuffer}; + use super::{mock_store, FlushPendingSnapshots, SnapshotBuffer}; use crate::snapshot_buffer::timelock_queue::Tick; use crate::{ AggregateConfig, AggregateId, E3id, EventContext, EventContextAccessors, EventContextSeq, @@ -381,6 +397,7 @@ mod tests { create_ec(7, 3), )) .await?; + buffer.send(FlushPendingSnapshots).await??; let batches = store.send(GetEvts).await?; assert_eq!( diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index fc5d7843e7..8e198ca6f0 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -177,6 +177,13 @@ pub trait SequenceIndex: Unpin + 'static { pub trait EventLog: Unpin + 'static { /// Append an event to the log, returning its sequence number fn append(&mut self, event: &InterfoldEvent) -> Result; + /// Flush buffered log and index data before a clean process exit. + /// + /// In-memory and test implementations may use the default no-op. Durable + /// implementations should override this and propagate I/O failures. + fn flush(&mut self) -> Result<()> { + Ok(()) + } /// Read all events starting from the given sequence number (inclusive) fn read_from(&self, from: u64) -> Box)>>; /// Read at most `limit` events starting from the given sequence number (inclusive). diff --git a/crates/sync/src/actors/sync.rs b/crates/sync/src/actors/sync.rs index 3f90fe1e66..0aea2fde67 100644 --- a/crates/sync/src/actors/sync.rs +++ b/crates/sync/src/actors/sync.rs @@ -72,8 +72,9 @@ pub async fn sync( info!("Replaying events to actors..."); // 4. Replay the EventStore events to all listeners (except effects). - // Skip infrastructure events (SyncEnded, EffectsEnabled, HistoricalEvmSyncStart) because - // they will be re-published by this sync process (steps 5, 8, 10). Replaying them here + // Skip lifecycle infrastructure events. SyncEnded, EffectsEnabled and sync-start events are + // re-published by this sync process; Shutdown belongs to the previous process and would stop + // freshly constructed actors. Replaying these here // would poison the EventBus deduplication window: the replayed event has the same // EventId (payload hash) as the one we publish later, causing the later event to be // silently dropped. This is critical for SyncEnded, if the EvmChainGateway never diff --git a/crates/sync/src/domain/sync_planner.rs b/crates/sync/src/domain/sync_planner.rs index 76844abafe..30382900f2 100644 --- a/crates/sync/src/domain/sync_planner.rs +++ b/crates/sync/src/domain/sync_planner.rs @@ -39,8 +39,9 @@ impl SyncPlanner { } } - /// True for events the sync flow re-publishes itself (steps 5/8/10) and therefore must not - /// replay from the EventStore. + /// True for process-lifecycle events that must not be replayed from the EventStore. Most are + /// re-published by sync; `Shutdown` is terminal for the previous process and replaying it would + /// stop freshly constructed actors during restart. pub fn is_infrastructure_event(event: &InterfoldEvent) -> bool { matches!( event.get_data(), @@ -48,6 +49,7 @@ impl SyncPlanner { | InterfoldEventData::EffectsEnabled(_) | InterfoldEventData::HistoricalEvmSyncStart(_) | InterfoldEventData::HistoricalNetSyncStart(_) + | InterfoldEventData::Shutdown(_) ) } @@ -105,7 +107,7 @@ mod tests { use super::*; use e3_events::{ E3Failed, E3RequestComplete, E3Stage, E3id, EffectsEnabled, EvmEventConfig, FailureReason, - HistoricalEvmSyncStart, InterfoldEvent, SyncEnded, Unsequenced, + HistoricalEvmSyncStart, InterfoldEvent, Shutdown, SyncEnded, Unsequenced, }; fn make_historical_evm_sync_start() -> HistoricalEvmSyncStart { @@ -129,14 +131,19 @@ mod tests { .data(make_historical_evm_sync_start()) .seq(3) .build(); + let shutdown = InterfoldEvent::::test_event("shutdown") + .data(Shutdown) + .seq(4) + .build(); let test_event = InterfoldEvent::::test_event("hello") .id(42) - .seq(4) + .seq(5) .build(); assert!(SyncPlanner::is_infrastructure_event(&sync_ended)); assert!(SyncPlanner::is_infrastructure_event(&effects_enabled)); assert!(SyncPlanner::is_infrastructure_event(&evm_sync_start)); + assert!(SyncPlanner::is_infrastructure_event(&shutdown)); assert!(!SyncPlanner::is_infrastructure_event(&test_event)); } diff --git a/dappnode/docker-compose.yml b/dappnode/docker-compose.yml index 63902add83..1bc0cb4026 100644 --- a/dappnode/docker-compose.yml +++ b/dappnode/docker-compose.yml @@ -7,6 +7,8 @@ services: UPSTREAM_VERSION: 0.1.8 image: 'ciphernode.interfold-ciphernode.public.dappnode.eth:0.1.8' restart: unless-stopped + # Allow the node's 30-second durability barrier to finish before Docker sends SIGKILL. + stop_grace_period: 45s volumes: - 'ciphernode_data:/data' ports: diff --git a/dappnode/tests/test-hardening.sh b/dappnode/tests/test-hardening.sh index aeedb1673d..08c615bed2 100644 --- a/dappnode/tests/test-hardening.sh +++ b/dappnode/tests/test-hardening.sh @@ -10,6 +10,9 @@ fail() { exit 1 } +grep -Fq 'stop_grace_period: 45s' "$ROOT_DIR/docker-compose.yml" \ + || fail "Docker stop grace period must exceed the node shutdown deadline" + assert_contains() { grep -Fq "$2" "$1" || fail "expected '$2' in $1" } diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml index 81bd56e286..07b6904be6 100644 --- a/deploy/docker-compose.yml +++ b/deploy/docker-compose.yml @@ -1,6 +1,7 @@ services: cn1: - image: { { IMAGE } } + image: '{{IMAGE}}' + stop_grace_period: 45s volumes: - ./cn1.yaml:/home/ciphernode/.config/interfold/config.yaml:ro - cn1-data:/home/ciphernode/.local/share/interfold @@ -18,7 +19,8 @@ services: - global-network cn2: - image: { { IMAGE } } + image: '{{IMAGE}}' + stop_grace_period: 45s volumes: - ./cn2.yaml:/home/ciphernode/.config/interfold/config.yaml:ro - cn2-data:/home/ciphernode/.local/share/interfold @@ -36,7 +38,8 @@ services: - global-network cn3: - image: { { IMAGE } } + image: '{{IMAGE}}' + stop_grace_period: 45s volumes: - ./cn3.yaml:/home/ciphernode/.config/interfold/config.yaml:ro - cn3-data:/home/ciphernode/.local/share/interfold @@ -54,7 +57,8 @@ services: - global-network cn4: - image: { { IMAGE } } + image: '{{IMAGE}}' + stop_grace_period: 45s depends_on: - cn1 volumes: From e72602e8d941a792781c094e205213f45bef1281 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 14:55:12 +0500 Subject: [PATCH 18/62] fix(config): preserve large finalization delays --- .../src/ciphernode_builder.rs | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 2dd0971295..3dd8fb598e 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -935,8 +935,7 @@ fn validate_chain_id(chain: &ChainConfig, actual_chain_id: u64) -> Result<()> { fn create_aggregate_delay(chain: &ChainConfig, actual_chain_id: u64) -> (AggregateId, Duration) { let aggregate_id = AggregateId::from_chain_id(Some(actual_chain_id)); let finalization_ms = chain.finalization_ms.unwrap_or(0); - let delay_us = finalization_ms * 1000; // ms → microseconds - (aggregate_id, Duration::from_micros(delay_us)) + (aggregate_id, Duration::from_millis(finalization_ms)) } /// Build delays configuration from chain providers @@ -1092,3 +1091,53 @@ async fn wait_for_evm_gateways(gateways: Vec) -> Result<( .await?; Ok(()) } + +#[cfg(test)] +mod tests { + use super::create_aggregate_delay; + use e3_config::{ + chain_config::ChainConfig, + contract::{Contract, ContractAddresses}, + rpc::RpcAuth, + }; + use std::time::Duration; + + fn chain_with_finalization_ms(finalization_ms: Option) -> ChainConfig { + let contract = || Contract::AddressOnly(Address::ZERO.to_string()); + ChainConfig { + enabled: Some(true), + name: "test".to_owned(), + rpc_url: "http://127.0.0.1:8545".to_owned(), + rpc_auth: RpcAuth::default(), + contracts: ContractAddresses { + interfold: contract(), + ciphernode_registry: contract(), + bonding_registry: contract(), + e3_program: None, + fee_token: None, + slashing_manager: None, + dkg_fold_attestation_verifier: None, + faucet: None, + }, + finalization_ms, + reorg_confirmations: None, + chain_id: Some(1), + } + } + + use alloy::primitives::Address; + + #[test] + fn aggregate_delay_preserves_large_millisecond_values_without_overflow() { + let (_, delay) = create_aggregate_delay(&chain_with_finalization_ms(Some(u64::MAX)), 1); + + assert_eq!(delay, Duration::from_millis(u64::MAX)); + } + + #[test] + fn aggregate_delay_defaults_to_zero() { + let (_, delay) = create_aggregate_delay(&chain_with_finalization_ms(None), 1); + + assert_eq!(delay, Duration::ZERO); + } +} From 49578d9905811a5a82adfaed8c532fd0ff9a56f0 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 15:00:45 +0500 Subject: [PATCH 19/62] fix(net): reject unauthorized internal gossip events --- crates/net/src/actors/net_sync_manager.rs | 54 +++++++++++++++++++++- crates/net/src/domain/event_translation.rs | 47 ++++++++++++++++--- crates/net/src/domain/sync_coordinator.rs | 45 +++++++++++++----- crates/tests/tests/integration.rs | 15 ++---- 4 files changed, 131 insertions(+), 30 deletions(-) diff --git a/crates/net/src/actors/net_sync_manager.rs b/crates/net/src/actors/net_sync_manager.rs index 8186b6c7a3..678811ce6a 100644 --- a/crates/net/src/actors/net_sync_manager.rs +++ b/crates/net/src/actors/net_sync_manager.rs @@ -528,7 +528,7 @@ async fn fetch_historical_events_for_aggregate( .retry_timeout(SYNC_FETCH_RETRY_TIMEOUT) .build(); - fetch_all_batched_events_with_budget::>( + let events = fetch_all_batched_events_with_budget::>( requester, PeerTarget::Random, aggregate_id, @@ -536,7 +536,31 @@ async fn fetch_historical_events_for_aggregate( 100, budget, ) - .await + .await?; + + validate_historical_events(aggregate_id, events) +} + +fn validate_historical_events( + aggregate_id: AggregateId, + events: Vec>, +) -> Result>> { + for event in &events { + if event.aggregate_id() != aggregate_id { + bail!( + "historical sync peer returned event for aggregate {} while fetching {}", + event.aggregate_id(), + aggregate_id + ); + } + if !EventTranslationService::is_forwardable_event(event) { + bail!( + "historical sync peer returned non-forwardable event type {}", + event.event_type() + ); + } + } + Ok(events) } async fn handle_sync_request_event( @@ -849,6 +873,32 @@ mod tests { .into_sequenced(2) } + fn remote_unsequenced(event: InterfoldEvent) -> InterfoldEvent { + event.clone_unsequenced().with_source(EventSource::Net) + } + + #[test] + fn historical_sync_rejects_non_forwardable_remote_events() { + let error = validate_historical_events( + AggregateId::new(0), + vec![remote_unsequenced(local_non_forwardable_event())], + ) + .unwrap_err(); + + assert!(error + .to_string() + .contains("non-forwardable event type TestEvent")); + } + + #[test] + fn historical_sync_rejects_events_from_another_aggregate() { + let event = remote_unsequenced(local_forwardable_event("1234")); + + let error = validate_historical_events(AggregateId::new(999), vec![event]).unwrap_err(); + + assert!(error.to_string().contains("while fetching 999")); + } + #[actix::test] async fn rebroadcast_only_gossips_forwardable_own_artifacts() { let system = EventSystem::new().with_fresh_bus(); diff --git a/crates/net/src/domain/event_translation.rs b/crates/net/src/domain/event_translation.rs index c66dcd8ee2..d17045ffb7 100644 --- a/crates/net/src/domain/event_translation.rs +++ b/crates/net/src/domain/event_translation.rs @@ -4,9 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::Result; +use anyhow::{ensure, Result}; use bloom::{BloomFilter, ASMS}; -use e3_events::{prelude::*, Event, InterfoldEvent, InterfoldEventData, Unsequenced}; +use e3_events::{prelude::*, Event, InterfoldEvent, InterfoldEventData, SeqState, Unsequenced}; use tracing::{trace, warn}; use crate::events::GossipData; @@ -36,7 +36,7 @@ impl EventTranslationService { /// Function to determine which events are allowed to be automatically broadcast to the /// network. Static so the same rule can be reused elsewhere (e.g. sync responses). - pub fn is_forwardable_event(event: &InterfoldEvent) -> bool { + pub fn is_forwardable_event(event: &InterfoldEvent) -> bool { matches!( event.get_data(), InterfoldEventData::DecryptionshareCreated(_) @@ -76,6 +76,11 @@ impl EventTranslationService { /// for dedup so it is not later rebroadcast. pub fn prepare_inbound(&mut self, data: GossipData) -> Result> { let event: InterfoldEvent = data.try_into()?; + ensure!( + Self::is_forwardable_event(&event), + "inbound gossip event type {} is not allowed on the protocol gossip channel", + event.event_type() + ); let id = event.id(); self.sent_events.insert(&id); Ok(event) @@ -85,7 +90,10 @@ impl EventTranslationService { #[cfg(test)] mod tests { use super::*; - use e3_events::{EventConstructorWithTimestamp, EventSource, TestEvent}; + use e3_events::{ + E3id, EventConstructorWithTimestamp, EventSource, PlaintextAggregated, TestEvent, + }; + use e3_utils::ArcBytes; fn local_test_event() -> InterfoldEvent { let unsequenced: InterfoldEvent = InterfoldEvent::new_with_timestamp( @@ -98,6 +106,22 @@ mod tests { unsequenced.into_sequenced(1) } + fn local_forwardable_event() -> InterfoldEvent { + let unsequenced: InterfoldEvent = InterfoldEvent::new_with_timestamp( + PlaintextAggregated { + e3_id: E3id::new("1", 1), + decrypted_output: vec![ArcBytes::from_bytes(&[1, 2, 3])], + decryption_aggregator_proofs: vec![], + } + .into(), + None, + 42, + None, + EventSource::Local, + ); + unsequenced.into_sequenced(1) + } + #[test] fn test_events_are_not_forwardable() { assert!(!EventTranslationService::is_forwardable_event( @@ -112,7 +136,7 @@ mod tests { } #[test] - fn inbound_gossip_round_trips_to_event() { + fn inbound_gossip_rejects_non_forwardable_internal_events() { let mut svc = EventTranslationService::new("topic"); let event: InterfoldEvent = InterfoldEvent::new_with_timestamp( TestEvent::new("fish", 7).into(), @@ -122,7 +146,18 @@ mod tests { EventSource::Local, ); let data: GossipData = event.clone().into_sequenced(3).try_into().unwrap(); + let error = svc.prepare_inbound(data).unwrap_err(); + assert!(error.to_string().contains("TestEvent")); + } + + #[test] + fn inbound_gossip_accepts_forwardable_protocol_events() { + let mut svc = EventTranslationService::new("topic"); + let expected = local_forwardable_event(); + let data: GossipData = expected.clone().try_into().unwrap(); + let decoded = svc.prepare_inbound(data).unwrap(); - assert_eq!(decoded.split().0, TestEvent::new("fish", 7).into()); + + assert_eq!(decoded.get_data(), expected.get_data()); } } diff --git a/crates/net/src/domain/sync_coordinator.rs b/crates/net/src/domain/sync_coordinator.rs index 608bdba8c4..5d5488ade7 100644 --- a/crates/net/src/domain/sync_coordinator.rs +++ b/crates/net/src/domain/sync_coordinator.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_events::{prelude::*, EventSource, InterfoldEvent, Unsequenced}; +use e3_events::{prelude::*, InterfoldEvent, Unsequenced}; use crate::domain::{ event_translation::EventTranslationService, @@ -125,18 +125,15 @@ pub fn build_sync_batch( let scan_limit = sync_scan_limit(fetch.limit()); let aggregate_id = fetch.aggregate_id(); - // Include Net events (received via gossip) and Local events that are gossip-forwardable. - // Without the Local check, a node's own gossip events would be excluded from sync responses, - // causing syncing peers to miss them. + // A remote-origin event is not trusted merely because it was persisted after gossip. Apply + // the same protocol allowlist to both local and relayed events so a peer cannot use historical + // sync to amplify an internal control event that an older or malicious node accepted. let scan_was_full = all_events.len() >= scan_limit; let mut events = Vec::with_capacity(limit); let mut last_scanned_ts = None; for event in all_events.into_iter().take(scan_limit) { last_scanned_ts = Some(event.ts()); - let is_forwardable = event.source() == EventSource::Net - || (event.source() == EventSource::Local - && EventTranslationService::is_forwardable_event(&event)); - if is_forwardable { + if EventTranslationService::is_forwardable_event(&event) { events.push(event.clone_unsequenced()); if events.len() == limit { break; @@ -169,7 +166,11 @@ pub fn build_sync_batch( #[cfg(test)] mod tests { use super::*; - use e3_events::{AggregateId, EventConstructorWithTimestamp, TestEvent}; + use e3_events::{ + AggregateId, E3id, EventConstructorWithTimestamp, EventSource, PlaintextAggregated, + TestEvent, + }; + use e3_utils::ArcBytes; #[test] fn no_peers_publishes_immediately_and_is_idempotent() { @@ -216,7 +217,23 @@ mod tests { fn net_event(ts: u128) -> InterfoldEvent { InterfoldEvent::::new_with_timestamp( - TestEvent::new("x", ts as u64).into(), + PlaintextAggregated { + e3_id: E3id::new(ts.to_string(), 1), + decrypted_output: vec![ArcBytes::from_bytes(&[1, 2, 3])], + decryption_aggregator_proofs: vec![], + } + .into(), + None, + ts, + None, + EventSource::Net, + ) + .into_sequenced(ts as u64) + } + + fn net_non_forwardable_event(ts: u128) -> InterfoldEvent { + InterfoldEvent::::new_with_timestamp( + TestEvent::new("remote-control", ts as u64).into(), None, ts, None, @@ -248,11 +265,15 @@ mod tests { #[test] fn build_sync_batch_filters_local_non_forwardable_and_marks_done() { let fetch = FetchEventsSince::new(AggregateId::new(1), 0, 10); - let outcome = build_sync_batch(vec![net_event(5), local_event(6)], &fetch); + let outcome = build_sync_batch( + vec![net_event(5), net_non_forwardable_event(6), local_event(7)], + &fetch, + ); let SyncBatchOutcome::Batch(batch) = outcome else { panic!("expected batch"); }; - // Only the Net event survives; the Local TestEvent is not forwardable. + // Only the allowlisted protocol event survives. Remote source does not make an internal + // TestEvent forwardable. assert_eq!(batch.events.len(), 1); assert!(matches!(batch.next, BatchCursor::Done)); } diff --git a/crates/tests/tests/integration.rs b/crates/tests/tests/integration.rs index f694de66d1..216b1a6b39 100644 --- a/crates/tests/tests/integration.rs +++ b/crates/tests/tests/integration.rs @@ -39,8 +39,6 @@ use e3_zk_prover::{VersionInfo, ZkBackend}; use fhe::bfv::PublicKey; use fhe_traits::{DeserializeParametrized, Serialize}; use num_bigint::BigUint; -use rand::SeedableRng; -use rand_chacha::ChaCha20Rng; use std::collections::HashSet; use std::ffi::OsString; use std::sync::atomic::{AtomicU64, Ordering}; @@ -2367,8 +2365,6 @@ async fn test_p2p_actor_forwards_events_to_network() -> Result<()> { #[actix::test] async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { - let seed = Seed(ChaCha20Rng::seed_from_u64(123).get_seed()); - // Setup elements in test let (cmd_tx, _) = mpsc::channel(100); // Transmit byte events to the network let (event_tx, event_rx) = broadcast::channel(100); // Receive byte events from the network @@ -2379,13 +2375,12 @@ async fn test_p2p_actor_forwards_events_to_bus() -> Result<()> { NetEventTranslator::setup(&bus, &cmd_tx, &Arc::new(event_rx), "mytopic"); // Capture messages from output on msgs vec - let event = E3Requested { + // Only protocol artifacts may cross the gossip trust boundary. Requests originate from the + // canonical chain listener, so use a genuinely forwardable result artifact here. + let event = PlaintextAggregated { e3_id: E3id::new("1235", 1), - threshold_m: 3, - threshold_n: 3, - seed, - params: ArcBytes::from_bytes(&[1, 2, 3, 4]), - ..E3Requested::default() + decrypted_output: vec![ArcBytes::from_bytes(&[1, 2, 3, 4])], + decryption_aggregator_proofs: vec![], }; // lets send an event from the network From e0c73abc764578f629d7dc1958637ccfaccc6647 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 15:15:06 +0500 Subject: [PATCH 20/62] fix(storage): add acknowledged atomic batch writes --- crates/ciphernode-builder/src/event_system.rs | 4 +- crates/data/src/data_store.rs | 125 +++++++++++++++++- crates/data/src/in_mem.rs | 31 ++++- crates/data/src/sled_db.rs | 48 ++++++- crates/data/src/sled_store.rs | 65 ++++++--- crates/events/src/data_events.rs | 22 ++- crates/events/src/snapshot_buffer/batch.rs | 2 +- .../src/snapshot_buffer/snapshot_buffer.rs | 3 +- 8 files changed, 267 insertions(+), 33 deletions(-) diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index eff69eb193..b524d594a9 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -409,9 +409,9 @@ impl Actor for NoopBatchReceiver { } impl Handler for NoopBatchReceiver { - type Result = (); + type Result = anyhow::Result<()>; fn handle(&mut self, _: InsertBatch, _: &mut Self::Context) -> Self::Result { - // do nothing + Ok(()) } } diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index a9c6b02c7d..1bc61d89f2 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -8,11 +8,14 @@ use std::borrow::Cow; use crate::{InMemStore, SledStore}; use actix::{Addr, Message, Recipient}; -use anyhow::anyhow; use anyhow::Context; use anyhow::Result; +use anyhow::{anyhow, ensure}; use e3_events::IntoKey; -use e3_events::{Flush, FlushPendingSnapshots, Get, Insert, InsertSync, Remove, SnapshotBuffer}; +use e3_events::{ + Flush, FlushPendingSnapshots, Get, Insert, InsertBatch, InsertBatchIfAbsent, InsertSync, + Remove, SnapshotBuffer, +}; use serde::{Deserialize, Serialize}; use tracing::error; @@ -48,6 +51,8 @@ pub struct DataStore { get: Recipient, insert: Recipient, insert_sync: Recipient, + insert_batch: Recipient, + insert_batch_if_absent: Recipient, remove: Recipient, flush: Recipient, flush_pending_snapshots: Option>, @@ -92,10 +97,62 @@ impl DataStore { let msg = InsertSync::new(&self.scope, serialized); self.insert_sync.send(msg).await??; - self.flush.send(Flush).await?; // Write sync will flush all pending writes + self.flush.send(Flush).await??; Ok(()) } + fn serialize_batch(&self, entries: I) -> Result> + where + K: IntoKey, + T: Serialize, + I: IntoIterator, + { + entries + .into_iter() + .map(|(key, value)| { + let scoped = self.scope(key); + let serialized = bincode::serialize(&value).with_context(|| { + let key = scoped.get_scope().unwrap_or(Cow::Borrowed("")); + anyhow!("Could not serialize batched value passed to {key}") + })?; + Ok(Insert::new(scoped.scope_bytes().to_vec(), serialized)) + }) + .collect() + } + + /// Atomically write and flush a group of same-typed values under this store's scopes. + pub async fn write_batch_sync(&self, entries: I) -> Result<()> + where + K: IntoKey, + T: Serialize, + I: IntoIterator, + { + let inserts = self.serialize_batch(entries)?; + ensure!(!inserts.is_empty(), "cannot write an empty storage batch"); + self.insert_batch.send(InsertBatch::new(inserts)).await??; + self.flush.send(Flush).await??; + Ok(()) + } + + /// Atomically write and flush a group only when every target scope is absent. + pub async fn write_batch_if_absent_sync(&self, entries: I) -> Result + where + K: IntoKey, + T: Serialize, + I: IntoIterator, + { + let inserts = self.serialize_batch(entries)?; + ensure!(!inserts.is_empty(), "cannot write an empty storage batch"); + let inserted = self + .insert_batch_if_absent + .send(InsertBatchIfAbsent::new(inserts)) + .await??; + if inserted { + self.flush.send(Flush).await??; + } + Ok(inserted) + } + /// Drain the snapshot buffer and durably close the backing store. pub async fn shutdown(&self) -> Result<()> { if let Some(flush_pending) = &self.flush_pending_snapshots { @@ -166,6 +223,8 @@ impl DataStore { get: self.get.clone(), insert: self.insert.clone(), insert_sync: self.insert_sync.clone(), + insert_batch: self.insert_batch.clone(), + insert_batch_if_absent: self.insert_batch_if_absent.clone(), remove: self.remove.clone(), scope, flush: self.flush.clone(), @@ -180,6 +239,8 @@ impl DataStore { get: self.get.clone(), insert: self.insert.clone(), insert_sync: self.insert_sync.clone(), + insert_batch: self.insert_batch.clone(), + insert_batch_if_absent: self.insert_batch_if_absent.clone(), remove: self.remove.clone(), scope: key.into_key(), flush: self.flush.clone(), @@ -198,6 +259,8 @@ impl DataStore { get: addr.clone().recipient(), insert: snapshot_buffer.into(), insert_sync: addr.clone().recipient(), + insert_batch: addr.clone().recipient(), + insert_batch_if_absent: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), @@ -215,6 +278,8 @@ impl DataStore { get: addr.clone().recipient(), insert: snapshot_buffer.clone().recipient(), insert_sync: addr.clone().recipient(), + insert_batch: addr.clone().recipient(), + insert_batch_if_absent: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), @@ -232,6 +297,8 @@ impl DataStore { get: addr.clone().recipient(), insert: snapshot_buffer.into(), insert_sync: addr.clone().recipient(), + insert_batch: addr.clone().recipient(), + insert_batch_if_absent: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), @@ -249,6 +316,8 @@ impl DataStore { get: addr.clone().recipient(), insert: snapshot_buffer.clone().recipient(), insert_sync: addr.clone().recipient(), + insert_batch: addr.clone().recipient(), + insert_batch_if_absent: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), @@ -263,6 +332,8 @@ impl DataStore { get: addr.clone().recipient(), insert: addr.clone().recipient(), insert_sync: addr.clone().recipient(), + insert_batch: addr.clone().recipient(), + insert_batch_if_absent: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), @@ -277,6 +348,8 @@ impl DataStore { get: addr.clone().recipient(), insert: addr.clone().recipient(), insert_sync: addr.clone().recipient(), + insert_batch: addr.clone().recipient(), + insert_batch_if_absent: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), @@ -293,6 +366,8 @@ impl From<&Addr> for DataStore { get: addr.clone().recipient(), insert: addr.clone().recipient(), insert_sync: addr.clone().recipient(), + insert_batch: addr.clone().recipient(), + insert_batch_if_absent: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), @@ -309,6 +384,8 @@ impl From<&Addr> for DataStore { get: addr.clone().recipient(), insert: addr.clone().recipient(), insert_sync: addr.clone().recipient(), + insert_batch: addr.clone().recipient(), + insert_batch_if_absent: addr.clone().recipient(), remove: addr.clone().recipient(), scope: vec![], flush: addr.clone().recipient(), @@ -333,4 +410,46 @@ mod tests { assert_eq!(scope, "//foo/bar/baz"); }); } + + #[actix::test] + async fn conditional_batch_write_is_all_or_nothing() -> Result<()> { + let addr = InMemStore::new(false).start(); + let store = DataStore::from(&addr); + store + .scope("existing") + .write_sync(b"original".to_vec()) + .await?; + + let inserted = store + .write_batch_if_absent_sync([ + ("existing", b"replacement".to_vec()), + ("missing", b"new".to_vec()), + ]) + .await?; + + assert!(!inserted); + assert_eq!( + store.scope("existing").read::>().await?, + Some(b"original".to_vec()) + ); + assert_eq!(store.scope("missing").read::>().await?, None); + Ok(()) + } + + #[actix::test] + async fn batch_write_persists_every_value() -> Result<()> { + let addr = InMemStore::new(false).start(); + let store = DataStore::from(&addr); + + store + .write_batch_sync([("first", vec![1_u8]), ("second", vec![2_u8])]) + .await?; + + assert_eq!(store.scope("first").read::>().await?, Some(vec![1])); + assert_eq!( + store.scope("second").read::>().await?, + Some(vec![2]) + ); + Ok(()) + } } diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index ccf846ff59..e661159cbe 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -10,7 +10,7 @@ use crate::{ }; use actix::{Actor, ActorContext, Handler, Message}; use anyhow::Result; -use e3_events::{Flush, Get, Insert, InsertBatch, InsertSync, Remove}; +use e3_events::{Flush, Get, Insert, InsertBatch, InsertBatchIfAbsent, InsertSync, Remove}; use e3_utils::MAILBOX_LIMIT; #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] @@ -62,7 +62,7 @@ impl Handler for InMemStore { } impl Handler for InMemStore { - type Result = (); + type Result = Result<()>; fn handle(&mut self, msg: InsertBatch, _: &mut Self::Context) -> Self::Result { for cmd in msg.commands() { self.store.insert( @@ -71,6 +71,29 @@ impl Handler for InMemStore { Some(DataOp::Insert(cmd.clone())), ); } + Ok(()) + } +} + +impl Handler for InMemStore { + type Result = Result; + + fn handle(&mut self, msg: InsertBatchIfAbsent, _: &mut Self::Context) -> Self::Result { + if msg + .commands() + .iter() + .any(|command| self.store.get(command.key()).is_some()) + { + return Ok(false); + } + for command in msg.commands() { + self.store.insert( + command.key().to_owned(), + command.value().to_owned(), + Some(DataOp::Insert(command.clone())), + ); + } + Ok(true) } } @@ -102,9 +125,9 @@ impl Handler for InMemStore { } impl Handler for InMemStore { - type Result = (); + type Result = Result<()>; fn handle(&mut self, _: Flush, _: &mut Self::Context) -> Self::Result { - // noop + Ok(()) } } diff --git a/crates/data/src/sled_db.rs b/crates/data/src/sled_db.rs index cbf824b206..17b3541539 100644 --- a/crates/data/src/sled_db.rs +++ b/crates/data/src/sled_db.rs @@ -7,7 +7,10 @@ use crate::sled_utils::{clear_all_caches, get_or_open_db_tree}; use anyhow::{Context, Result}; use e3_events::{Get, Insert, Remove}; -use sled::{transaction::ConflictableTransactionError, Tree}; +use sled::{ + transaction::{ConflictableTransactionError, TransactionError}, + Tree, +}; use std::path::PathBuf; pub struct SledDb { @@ -44,6 +47,29 @@ impl SledDb { Ok(()) } + /// Atomically insert `msgs` only if none of their keys exist. + pub fn insert_batch_if_absent(&mut self, msgs: &[Insert]) -> Result { + let result = self.db.transaction(|tx_db| { + for msg in msgs { + if tx_db.get(msg.key().as_slice())?.is_some() { + return Err(ConflictableTransactionError::Abort(())); + } + } + for msg in msgs { + tx_db.insert(msg.key().as_slice(), msg.value().to_vec())?; + } + Ok::<(), ConflictableTransactionError<()>>(()) + }); + + match result { + Ok(()) => Ok(true), + Err(TransactionError::Abort(())) => Ok(false), + Err(TransactionError::Storage(error)) => { + Err(error).context("Could not conditionally insert batch into db") + } + } + } + pub fn remove(&mut self, msg: Remove) -> Result<()> { self.db .remove(msg.key()) @@ -180,4 +206,24 @@ mod tests { Ok(()) } + + #[test] + fn conditional_batch_insert_never_overwrites_a_partial_key_set() -> Result<()> { + use tempfile::tempdir; + + let temp_dir = tempdir()?; + let db_path = temp_dir.path().join("test_conditional_batch.db"); + let mut db = SledDb::new(&db_path, "datastore")?; + db.insert(Insert::new("existing", b"original".to_vec()))?; + + let inserted = db.insert_batch_if_absent(&[ + Insert::new("existing", b"replacement".to_vec()), + Insert::new("missing", b"new".to_vec()), + ])?; + + assert!(!inserted); + assert_eq!(db.get(Get::new("existing"))?, Some(b"original".to_vec())); + assert_eq!(db.get(Get::new("missing"))?, None); + Ok(()) + } } diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 5ac4bbaec5..aa18ec4b61 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -8,7 +8,7 @@ use crate::{ShutdownStore, SledDb}; use actix::{Actor, ActorContext, Addr, Handler, ResponseFuture}; use anyhow::{Context, Result}; use e3_events::{BusHandle, EType, ErrorDispatcher, Flush, InterfoldEvent, Unsequenced}; -use e3_events::{Get, Insert, InsertBatch, InsertSync, Remove}; +use e3_events::{Get, Insert, InsertBatch, InsertBatchIfAbsent, InsertSync, Remove}; use e3_utils::MAILBOX_LIMIT; use std::path::PathBuf; use tracing::{error, info}; @@ -48,6 +48,12 @@ impl SledStore { self.write_failure = Some(format!("{error:#}")); } } + + fn report_write_failure(&mut self, error: &anyhow::Error) { + self.record_write_failure(error); + self.bus + .err(EType::Data, anyhow::anyhow!(format!("{error:#}"))); + } } impl Handler for SledStore { @@ -56,21 +62,39 @@ impl Handler for SledStore { fn handle(&mut self, event: Insert, _: &mut Self::Context) -> Self::Result { if let Some(ref mut db) = &mut self.db { if let Err(err) = db.insert(event) { - self.record_write_failure(&err); - self.bus.err(EType::Data, err) + self.report_write_failure(&err); } } } } impl Handler for SledStore { - type Result = (); + type Result = Result<()>; fn handle(&mut self, event: InsertBatch, _: &mut Self::Context) -> Self::Result { - if let Some(ref mut db) = &mut self.db { - if let Err(err) = db.insert_batch(event.commands()) { - self.record_write_failure(&err); - self.bus.err(EType::Data, err) + let Some(ref mut db) = &mut self.db else { + anyhow::bail!("SledStore is closed"); + }; + if let Err(error) = db.insert_batch(event.commands()) { + self.report_write_failure(&error); + return Err(error); + } + Ok(()) + } +} + +impl Handler for SledStore { + type Result = Result; + + fn handle(&mut self, event: InsertBatchIfAbsent, _: &mut Self::Context) -> Self::Result { + let Some(ref mut db) = &mut self.db else { + anyhow::bail!("SledStore is closed"); + }; + match db.insert_batch_if_absent(event.commands()) { + Ok(inserted) => Ok(inserted), + Err(error) => { + self.report_write_failure(&error); + Err(error) } } } @@ -80,11 +104,12 @@ impl Handler for SledStore { type Result = Result<()>; fn handle(&mut self, event: InsertSync, _: &mut Self::Context) -> Self::Result { - if let Some(ref mut db) = &mut self.db { - if let Err(error) = db.insert(event.into()) { - self.record_write_failure(&error); - return Err(error); - } + let Some(ref mut db) = &mut self.db else { + anyhow::bail!("SledStore is closed"); + }; + if let Err(error) = db.insert(event.into()) { + self.report_write_failure(&error); + return Err(error); } Ok(()) } @@ -123,14 +148,16 @@ impl Handler for SledStore { } impl Handler for SledStore { - type Result = (); + type Result = Result<()>; fn handle(&mut self, _: Flush, _: &mut Self::Context) -> Self::Result { - if let Some(ref db) = self.db { - if let Err(err) = db.flush() { - self.record_write_failure(&err); - self.bus.err(EType::Data, err) - } + let Some(ref db) = self.db else { + anyhow::bail!("SledStore is closed"); + }; + if let Err(error) = db.flush() { + self.report_write_failure(&error); + return Err(error); } + Ok(()) } } diff --git a/crates/events/src/data_events.rs b/crates/events/src/data_events.rs index 96020db3ed..91ff48518f 100644 --- a/crates/events/src/data_events.rs +++ b/crates/events/src/data_events.rs @@ -51,7 +51,7 @@ impl Insert { } #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] -#[rtype(result = "()")] +#[rtype(result = "Result<()>")] pub struct InsertBatch(pub Vec); impl InsertBatch { pub fn new(commands: Vec) -> Self { @@ -63,6 +63,24 @@ impl InsertBatch { } } +/// Atomically insert every command only when all target keys are absent. +/// +/// Returns `true` when the batch was inserted and `false` when at least one key +/// already existed. Storage failures are returned to the caller. +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] +#[rtype(result = "Result")] +pub struct InsertBatchIfAbsent(pub Vec); + +impl InsertBatchIfAbsent { + pub fn new(commands: Vec) -> Self { + Self(commands) + } + + pub fn commands(&self) -> &[Insert] { + &self.0 + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash)] #[rtype(result = "Result<()>")] pub struct InsertSync(pub Vec, pub Vec); @@ -113,5 +131,5 @@ impl Remove { } #[derive(Message)] -#[rtype(result = "()")] +#[rtype(result = "Result<()>")] pub struct Flush; diff --git a/crates/events/src/snapshot_buffer/batch.rs b/crates/events/src/snapshot_buffer/batch.rs index e0f40a090d..b66d375104 100644 --- a/crates/events/src/snapshot_buffer/batch.rs +++ b/crates/events/src/snapshot_buffer/batch.rs @@ -68,7 +68,7 @@ impl Handler for Batch { if !inserts.is_empty() { db.send(InsertBatch::new(inserts)) .await - .context("snapshot destination stopped before accepting its final batch")?; + .context("snapshot destination stopped before accepting its final batch")??; } Ok(()) }) diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index 40bba6eba0..fd378db0b6 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -194,10 +194,11 @@ mod mock_store { } impl Handler for MockStore { - type Result = (); + type Result = anyhow::Result<()>; fn handle(&mut self, msg: InsertBatch, _: &mut Self::Context) -> Self::Result { self.evts.push(msg); + Ok(()) } } From bc62534c02bdefe8480487bb8919accab20352e9 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 15:15:11 +0500 Subject: [PATCH 21/62] fix(wallet): preserve autowallet node identities --- crates/entrypoint/src/wallet/set.rs | 161 +++++++++++++++++++++++++--- 1 file changed, 147 insertions(+), 14 deletions(-) diff --git a/crates/entrypoint/src/wallet/set.rs b/crates/entrypoint/src/wallet/set.rs index 07715d6a9c..2e40f59489 100644 --- a/crates/entrypoint/src/wallet/set.rs +++ b/crates/entrypoint/src/wallet/set.rs @@ -9,6 +9,8 @@ use alloy_primitives::Address; use anyhow::{anyhow, Result}; use e3_config::AppConfig; use e3_crypto::Cipher; +use e3_data::Repositories; +use e3_events::StoreKeys; use e3_evm::EthPrivateKeyRepositoryFactory; use e3_net::NetRepositoryFactory; use libp2p::{identity::Keypair, PeerId}; @@ -29,18 +31,23 @@ pub async fn execute(config: &AppConfig, input: Zeroizing) -> Result<(Ad let (encrypted_private_key, encrypted_keypair, address, peer_id) = process_key(&cipher, input)?; - // Save the encrypted keys let repositories = get_repositories(config)?; - repositories - .eth_private_key() - .write_sync(&encrypted_private_key) - .await?; + write_key_pair(&repositories, encrypted_private_key, encrypted_keypair).await?; + Ok((address, peer_id)) +} +async fn write_key_pair( + repositories: &Repositories, + encrypted_private_key: Vec, + encrypted_keypair: Vec, +) -> Result<()> { repositories - .libp2p_keypair() - .write_sync(&encrypted_keypair) - .await?; - Ok((address, peer_id)) + .store + .write_batch_sync([ + (StoreKeys::eth_private_key(), encrypted_private_key), + (StoreKeys::libp2p_keypair(), encrypted_keypair), + ]) + .await } fn process_key( @@ -59,24 +66,76 @@ fn process_key( } pub async fn autowallet(config: &AppConfig) -> Result<()> { + let cipher = Cipher::from_file(config.key_file()).await?; + let repositories = get_repositories(config)?; + ensure_autowallet(&repositories, &cipher).await?; + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum AutowalletOutcome { + Existing, + Generated, +} + +async fn read_key_pair(repositories: &Repositories) -> Result<(Option>, Option>)> { + let private_key = repositories.eth_private_key().read().await?; + let network_key = repositories.libp2p_keypair().read().await?; + Ok((private_key, network_key)) +} + +async fn ensure_autowallet( + repositories: &Repositories, + cipher: &Cipher, +) -> Result { + match read_key_pair(repositories).await? { + (Some(_), Some(_)) => return Ok(AutowalletOutcome::Existing), + (Some(_), None) | (None, Some(_)) => { + anyhow::bail!( + "wallet storage is inconsistent: operator and libp2p keys must either both exist or both be absent" + ); + } + (None, None) => {} + } + let mut bytes = generate_random_bytes(32); let input = Zeroizing::new(hex::encode(&bytes)); bytes.zeroize(); - execute(config, input).await?; - Ok(()) + let (encrypted_private_key, encrypted_keypair, _, _) = process_key(cipher, input)?; + let inserted = repositories + .store + .write_batch_if_absent_sync([ + (StoreKeys::eth_private_key(), encrypted_private_key), + (StoreKeys::libp2p_keypair(), encrypted_keypair), + ]) + .await?; + if inserted { + return Ok(AutowalletOutcome::Generated); + } + + // Another process may have initialized the pair between our read and conditional insert. + // Accept only a complete pair; never repair a half-written identity by rotating one side. + match read_key_pair(repositories).await? { + (Some(_), Some(_)) => Ok(AutowalletOutcome::Existing), + _ => anyhow::bail!( + "wallet storage became inconsistent during initialization; refusing to rotate keys" + ), + } } #[cfg(test)] mod tests { use super::*; + use e3_data::Repositories; + + const TEST_PRIVATE_KEY: &str = + "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; #[tokio::test] async fn test_process_key() -> Result<()> { let cipher = Cipher::from_password("test_password").await?; // Hardhat default private key - let input = Zeroizing::new( - "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80".to_string(), - ); + let input = Zeroizing::new(TEST_PRIVATE_KEY.to_string()); let (encrypted_private_key, encrypted_keypair, address, peer_id) = process_key(&cipher, input)?; @@ -94,4 +153,78 @@ mod tests { Ok(()) } + + #[actix::test] + async fn autowallet_preserves_an_existing_operator_and_peer_identity() -> Result<()> { + let repositories = Repositories::in_mem(); + let cipher = Cipher::from_password("test_password").await?; + let (private_key, network_key, expected_address, expected_peer_id) = + process_key(&cipher, Zeroizing::new(TEST_PRIVATE_KEY.to_owned()))?; + write_key_pair(&repositories, private_key.clone(), network_key.clone()).await?; + + let outcome = ensure_autowallet(&repositories, &cipher).await?; + + assert_eq!(outcome, AutowalletOutcome::Existing); + assert_eq!( + read_key_pair(&repositories).await?, + (Some(private_key.clone()), Some(network_key.clone())) + ); + let decrypted_private_key = cipher.decrypt_data(&private_key)?; + let address = + PrivateKeySigner::from_bytes(&FixedBytes::<32>::from_slice(&decrypted_private_key))? + .address(); + let mut decrypted_network_key = cipher.decrypt_data(&network_key)?; + let keypair: Keypair = + libp2p::identity::ed25519::Keypair::try_from_bytes(&mut decrypted_network_key)?.into(); + assert_eq!(address, expected_address); + assert_eq!(PeerId::from(keypair.public()), expected_peer_id); + Ok(()) + } + + #[actix::test] + async fn autowallet_creates_both_keys_when_storage_is_empty() -> Result<()> { + let repositories = Repositories::in_mem(); + let cipher = Cipher::from_password("test_password").await?; + + let outcome = ensure_autowallet(&repositories, &cipher).await?; + let (private_key, network_key) = read_key_pair(&repositories).await?; + + assert_eq!(outcome, AutowalletOutcome::Generated); + assert!(private_key.is_some()); + assert!(network_key.is_some()); + Ok(()) + } + + #[actix::test] + async fn autowallet_rejects_half_initialized_storage_without_mutation() -> Result<()> { + let repositories = Repositories::in_mem(); + let cipher = Cipher::from_password("test_password").await?; + let sentinel = vec![1, 2, 3, 4]; + repositories.eth_private_key().write_sync(&sentinel).await?; + + let error = ensure_autowallet(&repositories, &cipher).await.unwrap_err(); + + assert!(error.to_string().contains("wallet storage is inconsistent")); + assert_eq!(read_key_pair(&repositories).await?, (Some(sentinel), None)); + Ok(()) + } + + #[actix::test] + async fn concurrent_autowallet_initialization_creates_only_one_pair() -> Result<()> { + let repositories = Repositories::in_mem(); + let cipher = Cipher::from_password("test_password").await?; + + let (first, second) = tokio::join!( + ensure_autowallet(&repositories, &cipher), + ensure_autowallet(&repositories, &cipher) + ); + + let outcomes = [first?, second?]; + assert!(outcomes.contains(&AutowalletOutcome::Generated)); + assert!(outcomes.contains(&AutowalletOutcome::Existing)); + let (private_key, network_key) = read_key_pair(&repositories).await?; + assert!(private_key.is_some()); + assert!(network_key.is_some()); + Ok(()) + } } From c124ba8b3333d10fce8cb44c22c6f401a7dffbe2 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 15:24:53 +0500 Subject: [PATCH 22/62] fix(storage): isolate and acknowledge snapshot batches --- crates/events/src/snapshot_buffer/batch.rs | 26 +-- .../src/snapshot_buffer/batch_router.rs | 194 +++++++++++------ .../src/snapshot_buffer/snapshot_buffer.rs | 204 +++++++++++++----- .../src/snapshot_buffer/timelock_queue.rs | 39 ++-- 4 files changed, 311 insertions(+), 152 deletions(-) diff --git a/crates/events/src/snapshot_buffer/batch.rs b/crates/events/src/snapshot_buffer/batch.rs index b66d375104..5465417705 100644 --- a/crates/events/src/snapshot_buffer/batch.rs +++ b/crates/events/src/snapshot_buffer/batch.rs @@ -5,15 +5,11 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use super::FlushPendingSnapshots; -use actix::{Actor, ActorContext, Addr, AsyncContext, Handler, Message, Recipient, ResponseFuture}; +use actix::{Actor, ActorContext, Addr, Handler, Recipient, ResponseFuture}; use anyhow::{Context, Result}; use e3_utils::MAILBOX_LIMIT; -use crate::{Die, Insert, InsertBatch}; - -#[derive(Message)] -#[rtype(result = "()")] -pub struct Flush; +use crate::{Insert, InsertBatch}; pub struct Batch { inserts: Vec, @@ -46,17 +42,6 @@ impl Handler for Batch { } } -impl Handler for Batch { - type Result = (); - fn handle(&mut self, _: Flush, ctx: &mut Self::Context) -> Self::Result { - let inserts = std::mem::take(&mut self.inserts); - if !inserts.is_empty() { - self.db.do_send(InsertBatch::new(inserts)); - } - ctx.notify(Die); - } -} - impl Handler for Batch { type Result = ResponseFuture>; @@ -74,10 +59,3 @@ impl Handler for Batch { }) } } - -impl Handler for Batch { - type Result = (); - fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { - ctx.stop(); - } -} diff --git a/crates/events/src/snapshot_buffer/batch_router.rs b/crates/events/src/snapshot_buffer/batch_router.rs index 53df14a0e2..68b2317738 100644 --- a/crates/events/src/snapshot_buffer/batch_router.rs +++ b/crates/events/src/snapshot_buffer/batch_router.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. use super::{ - batch::{Batch, Flush}, + batch::Batch, timelock_queue::{Clock, StartTimelock}, AggregateConfig, FlushPendingSnapshots, UpdateDestination, }; @@ -12,32 +12,60 @@ use crate::{ AggregateId, EventContextAccessors, EventContextSeq, EventType, Insert, InsertBatch, InterfoldEvent, Sequenced, StoreKeys, }; -use actix::{Actor, Addr, Handler, Message, Recipient, ResponseFuture}; -use anyhow::{Context, Result}; +use actix::{ + Actor, ActorFutureExt, Addr, AsyncContext, Handler, Message, Recipient, ResponseFuture, + WrapFuture, +}; +use anyhow::Result; use e3_utils::MAILBOX_LIMIT; use std::{collections::HashMap, sync::Arc, time::Duration}; use tracing::{debug, error}; type Seq = u64; +/// Snapshot batches are scoped by both aggregate and per-aggregate sequence. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub(super) struct SnapshotKey { + aggregate_id: AggregateId, + seq: Seq, +} + +impl SnapshotKey { + pub(super) fn new(aggregate_id: AggregateId, seq: Seq) -> Self { + Self { aggregate_id, seq } + } + + pub(super) fn aggregate_id(self) -> AggregateId { + self.aggregate_id + } + + pub(super) fn seq(self) -> Seq { + self.seq + } +} + #[derive(Message)] #[rtype(result = "()")] -pub struct FlushSeq(pub Seq); +pub struct FlushSeq(pub(super) SnapshotKey); impl FlushSeq { pub fn seq(&self) -> u64 { - self.0 + self.0.seq() + } + + pub fn aggregate_id(&self) -> AggregateId { + self.0.aggregate_id() } } pub struct BatchRouter { config: AggregateConfig, - aggregates: HashMap, - batches: HashMap>, + batches: HashMap>, block_height_seen: HashMap, timelock_queue: Recipient, db: Recipient, clock: Arc, + write_failure: Option, } impl Actor for BatchRouter { @@ -70,12 +98,12 @@ impl BatchRouter { ) -> Self { Self { batches: HashMap::new(), - aggregates: HashMap::new(), config: config.clone(), timelock_queue: timelock_queue.into(), block_height_seen: HashMap::new(), db: db.into(), clock, + write_failure: None, } } @@ -90,19 +118,12 @@ impl BatchRouter { highest } - /// Force-flush every open batch to disk and clear routing state. Used on - /// shutdown so debounced batches that have not yet hit their timelock are - /// committed before the process exits, instead of being lost in the - /// ~500ms durability window (H2/GF-5). - fn flush_all(&mut self) { - if self.batches.is_empty() { - return; + fn record_write_failure(&mut self, error: anyhow::Error) { + let error = format!("{error:#}"); + error!(%error, "Snapshot batch flush failed"); + if self.write_failure.is_none() { + self.write_failure = Some(error); } - debug!("Force-flushing {} open batch(es)", self.batches.len()); - for (_, batch) in self.batches.drain() { - batch.do_send(Flush); - } - self.aggregates.clear(); } } @@ -118,16 +139,22 @@ impl Handler for BatchRouter { }; // Route to existing batch, or fall back to disk - match self.batches.get(&ctx.seq()) { + let key = SnapshotKey::new(ctx.aggregate_id(), ctx.seq()); + match self.batches.get(&key) { Some(batch) => { - debug!("Forwarding to batch actor for seq={}", ctx.seq()); + debug!( + aggregate = %key.aggregate_id(), + seq = key.seq(), + "Forwarding insert to snapshot batch" + ); batch.do_send(msg); } // This must mean that this insert is late None => { debug!( - "No batch available for seq={} assuming this is late. Flushing to disk.", - ctx.seq() + aggregate = %key.aggregate_id(), + seq = key.seq(), + "No snapshot batch is open; flushing late insert directly" ); self.db.do_send(InsertBatch::new(vec![msg])); } @@ -138,40 +165,46 @@ impl Handler for BatchRouter { impl Handler> for BatchRouter { type Result = (); fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { - // On shutdown, force every still-open batch to disk before the process - // exits. The batch that carries the Shutdown event itself is committed - // by the normal path below; this drains any earlier debounced batches - // whose timelock has not yet fired (H2/GF-5). + // Keep every open batch registered until the acknowledged + // `FlushPendingSnapshots` fence drains it. Enqueueing child flushes here and clearing the + // map lets the later fence return before the destination store has handled them. if msg.event_type_enum() == EventType::Shutdown { - self.flush_all(); + debug!( + open_batches = self.batches.len(), + "Shutdown observed; snapshot batches await the acknowledged drain" + ); return; } let ec = msg.get_ctx(); - let prev_seq = ec.seq() - 1; - if self.batches.contains_key(&prev_seq) { - let Some(prev_agg) = self.aggregates.get(&prev_seq) else { - error!( - "invariant violation: prev_agg must exist if batches has a batch for seq={}", - prev_seq + let agg_id = ec.aggregate_id(); + if let Some(prev_seq) = ec.seq().checked_sub(1) { + let previous = SnapshotKey::new(agg_id, prev_seq); + if self.batches.contains_key(&previous) { + debug!( + aggregate = %agg_id, + seq = prev_seq, + "Scheduling previous snapshot batch" ); - return; - }; + let delay = self.config.get_delay(&agg_id); - debug!( - "Preparing timelock to clear batch for seq={}, agg={}", - prev_seq, prev_agg - ); - let delay = self.config.get_delay(prev_agg); + let now = Duration::from_micros(self.clock.now_micros()); - let now = Duration::from_micros(self.clock.now_micros()); - - self.timelock_queue - .do_send(StartTimelock::new(prev_seq, now, delay)); + self.timelock_queue + .do_send(StartTimelock::new(previous, now, delay)); + } } - debug!("Creating batch for {}", ec.seq()); - let agg_id = ec.aggregate_id(); + let key = SnapshotKey::new(agg_id, ec.seq()); + if self.batches.contains_key(&key) { + error!( + aggregate = %agg_id, + seq = ec.seq(), + "Snapshot batch already exists for event context; retaining the original batch" + ); + return; + } + debug!(aggregate = %agg_id, seq = ec.seq(), "Creating snapshot batch"); let highest_block = self.get_highest_block(agg_id, ec.block()); let batch = Batch::spawn( self.db.clone(), @@ -194,20 +227,40 @@ impl Handler> for BatchRouter { ], ); - self.batches.insert(ec.seq(), batch); - self.aggregates.insert(ec.seq(), ec.aggregate_id()); + self.batches.insert(key, batch); } } impl Handler for BatchRouter { type Result = (); - fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { - debug!("Flushing sequence... {}", msg.seq()); - if let Some(batch) = self.batches.get(&msg.seq()) { - batch.do_send(Flush); - self.batches.remove(&msg.seq()); - self.aggregates.remove(&msg.seq()); + fn handle(&mut self, msg: FlushSeq, ctx: &mut Self::Context) -> Self::Result { + let key = msg.0; + debug!( + aggregate = %key.aggregate_id(), + seq = key.seq(), + "Flushing snapshot batch" + ); + let Some(batch) = self.batches.remove(&key) else { + return; + }; + + let flush = async move { + batch + .send(FlushPendingSnapshots) + .await + .map_err(anyhow::Error::from)??; + Ok::<(), anyhow::Error>(()) } + .into_actor(self) + .map(|result, actor, _| { + if let Err(error) = result { + actor.record_write_failure(error); + } + }); + + // Preserve event/insert ordering and ensure the shutdown fence cannot overtake a normal + // timelock flush that is already in the router mailbox. + ctx.wait(flush); } } @@ -216,13 +269,32 @@ impl Handler for BatchRouter { fn handle(&mut self, _: FlushPendingSnapshots, _: &mut Self::Context) -> Self::Result { let batches: Vec<_> = self.batches.drain().map(|(_, batch)| batch).collect(); - self.aggregates.clear(); + let db = self.db.clone(); + let previous_failure = self.write_failure.take(); Box::pin(async move { + let mut failures = previous_failure.into_iter().collect::>(); for batch in batches { - batch - .send(FlushPendingSnapshots) - .await - .context("snapshot batch stopped before its final flush")??; + match batch.send(FlushPendingSnapshots).await { + Ok(Ok(())) => {} + Ok(Err(error)) => failures.push(format!("{error:#}")), + Err(error) => failures.push(format!( + "snapshot batch stopped before its final flush: {error}" + )), + } + } + + // Destination-mailbox barrier for late/direct inserts that were already sent by this + // router before the shutdown fence. The empty transactional batch has no data effect. + match db.send(InsertBatch::new(vec![])).await { + Ok(Ok(())) => {} + Ok(Err(error)) => failures.push(format!("{error:#}")), + Err(error) => failures.push(format!( + "snapshot destination stopped before its final barrier: {error}" + )), + } + + if !failures.is_empty() { + anyhow::bail!("snapshot drain failed: {}", failures.join("; ")); } Ok(()) }) diff --git a/crates/events/src/snapshot_buffer/snapshot_buffer.rs b/crates/events/src/snapshot_buffer/snapshot_buffer.rs index fd378db0b6..b9564e9366 100644 --- a/crates/events/src/snapshot_buffer/snapshot_buffer.rs +++ b/crates/events/src/snapshot_buffer/snapshot_buffer.rs @@ -197,7 +197,9 @@ mod mock_store { type Result = anyhow::Result<()>; fn handle(&mut self, msg: InsertBatch, _: &mut Self::Context) -> Self::Result { - self.evts.push(msg); + if !msg.commands().is_empty() { + self.evts.push(msg); + } Ok(()) } } @@ -221,14 +223,54 @@ mod tests { EventId, EventSource, Insert, InsertBatch, InterfoldEvent, Sequenced, Shutdown, SyncEnded, TestEvent, }; - use actix::Actor; - use anyhow::Result; + use actix::{Actor, Handler, ResponseFuture}; + use anyhow::{Context, Result}; use e3_test_helpers::with_tracing; - use std::collections::HashMap; - use std::sync::Arc; + use std::collections::{HashMap, HashSet}; + use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }; use std::time::Duration; + use tokio::sync::oneshot; use tracing::info; + struct GatedStore { + entered: Option>, + release: Option>, + write_completed: Arc, + } + + impl Actor for GatedStore { + type Context = actix::Context; + } + + impl Handler for GatedStore { + type Result = ResponseFuture>; + + fn handle(&mut self, msg: InsertBatch, _: &mut Self::Context) -> Self::Result { + if msg.commands().is_empty() { + return Box::pin(async { Ok(()) }); + } + + let entered = self.entered.take(); + let release = self.release.take(); + let write_completed = self.write_completed.clone(); + Box::pin(async move { + entered + .context("gated store received more than one data batch")? + .send(()) + .map_err(|_| anyhow::anyhow!("flush test stopped before the write began"))?; + release + .context("gated store received more than one data batch")? + .await + .context("flush test dropped the destination write gate")?; + write_completed.store(true, Ordering::SeqCst); + Ok(()) + }) + } + } + fn create_ec(ag: usize, seq: u64) -> EventContext { EventContext::new_origin( EventId::hash(1), @@ -264,71 +306,43 @@ mod tests { let (buffer, timelock) = SnapshotBuffer::with_clock(config, store.clone(), clock.clone(), None)?; - buffer - .send(InterfoldEvent::from_data_ec( - SyncEnded::new().into(), - create_ec(0, 9), - )) - .await?; - - info!("TimelockQueue should be empty"); - timelock.send(Tick).await?; - - let ec = create_ec(23, 10); - let interfold_10 = create_event(&ec); + let ec = create_ec(23, 1); + let aggregate_23_first = create_event(&ec); - let inserts_10 = [ + let aggregate_23_inserts = [ Insert::new_with_context("one", b"one".to_vec(), ec.clone()), Insert::new_with_context("two", b"two".to_vec(), ec.clone()), ]; - let ec = create_ec(1, 11); - let interfold_11 = create_event(&ec); + let ec = create_ec(1, 1); + let aggregate_1_first = create_event(&ec); - let inserts_11 = [ + let aggregate_1_inserts = [ Insert::new_with_context("one", b"one".to_vec(), ec.clone()), Insert::new_with_context("two", b"two".to_vec(), ec.clone()), ]; - let ec = create_ec(0, 12); - let interfold_12 = create_event(&ec); - - // send event 10 - buffer.send(interfold_10).await?; - - info!("TimelockQueue should hold all seq=9 inserts"); - timelock.send(Tick).await?; - - // send the first insert for seq 10 - buffer.send(inserts_10[0].clone()).await?; + buffer.send(aggregate_23_first).await?; + buffer.send(aggregate_23_inserts[0].clone()).await?; + // The next sequence for the same aggregate starts its predecessor's timelock. + buffer.send(create_event(&create_ec(23, 2))).await?; + // State written during the grace period still belongs to aggregate 23, sequence 1. + buffer.send(aggregate_23_inserts[1].clone()).await?; - // send event 11 - info!("Sending event seq=11 this should start the timelock for all the seq=10 inserts"); - buffer.send(interfold_11).await?; - - // send a late insert for 10 - buffer.send(inserts_10[1].clone()).await?; - - // send the other inserts for 11 - buffer.send(inserts_11[0].clone()).await?; - buffer.send(inserts_11[1].clone()).await?; - - // send event 12 - info!("Sending event seq=12 this should start the timelock for all the seq=11 inserts"); - buffer.send(interfold_12).await?; + buffer.send(aggregate_1_first).await?; + buffer.send(aggregate_1_inserts[0].clone()).await?; + buffer.send(aggregate_1_inserts[1].clone()).await?; + buffer.send(create_event(&create_ec(1, 2))).await?; // Nothing happens as there has not been enough delay info!("Clock=1020 : Checking for events but there should be nothing that has flushed..."); clock.set(Duration::from_micros(1020)); timelock.send(Tick).await?; let batches = store.send(GetEvts).await?; - // assert_eq!(0, batches.len()); - assert_eq!(1, batches.len()); - let InsertBatch(inserts) = batches.first().unwrap(); - assert_eq!(3, inserts.len()); // Have sequence,block and ts written as inserts + assert_eq!(0, batches.len()); // Time is up so lets flush aggregate 23 (but not aggregate 1) - info!("Clock=1030 : Checking for events Tick should flush batch 10..."); + info!("Clock=1030 : aggregate 23 sequence 1 should flush..."); clock.set(Duration::from_micros(1030)); timelock.send(Tick).await?; let batches = store.send(GetEvts).await?; @@ -344,7 +358,7 @@ mod tests { assert_eq!(0, batches.len()); // Time is up so lets flush aggregate 1 - info!("Clock=1060 : should have all aggregate 1 changes in batch 11..."); + info!("Clock=1060 : aggregate 1 sequence 1 should flush..."); clock.set(Duration::from_micros(1060)); timelock.send(Tick).await?; let batches = store.send(GetEvts).await?; @@ -355,6 +369,92 @@ mod tests { Ok(()) } + #[actix::test] + async fn equal_sequences_from_different_aggregates_use_distinct_batches() -> Result<()> { + let config = &AggregateConfig::new(HashMap::from([ + (AggregateId::new(1), Duration::from_secs(60)), + (AggregateId::new(2), Duration::from_secs(60)), + ])); + let store = mock_store::MockStore::default().start(); + let clock = Arc::new(MockClock::new(1000)); + let (buffer, _) = SnapshotBuffer::with_clock(config, store.clone(), clock.clone(), None)?; + + let first = create_ec(1, 1); + let second = create_ec(2, 1); + buffer.send(create_event(&first)).await?; + buffer.send(create_event(&second)).await?; + buffer + .send(Insert::new_with_context("aggregate-one", vec![1], first)) + .await?; + buffer + .send(Insert::new_with_context("aggregate-two", vec![2], second)) + .await?; + + buffer.send(FlushPendingSnapshots).await??; + + let batches = store.send(GetEvts).await?; + assert_eq!(batches.len(), 2); + let aggregate_ids = batches + .iter() + .flat_map(|batch| batch.commands()) + .filter_map(Insert::ctx) + .map(EventContextAccessors::aggregate_id) + .collect::>(); + assert_eq!( + aggregate_ids, + HashSet::from([AggregateId::new(1), AggregateId::new(2)]) + ); + Ok(()) + } + + #[actix::test] + async fn final_flush_waits_for_destination_acknowledgement() -> Result<()> { + let config = &AggregateConfig::new(HashMap::from([( + AggregateId::new(1), + Duration::from_secs(60), + )])); + let (entered_tx, entered_rx) = oneshot::channel(); + let (release_tx, release_rx) = oneshot::channel(); + let write_completed = Arc::new(AtomicBool::new(false)); + let store = GatedStore { + entered: Some(entered_tx), + release: Some(release_rx), + write_completed: write_completed.clone(), + } + .start(); + let clock = Arc::new(MockClock::new(1000)); + let (buffer, _) = SnapshotBuffer::with_clock(config, store, clock, None)?; + + let context = create_ec(1, 1); + buffer.send(create_event(&context)).await?; + buffer + .send(Insert::new_with_context("state", vec![1], context)) + .await?; + + let mut flush = Box::pin(buffer.send(FlushPendingSnapshots)); + tokio::select! { + result = &mut flush => anyhow::bail!("final flush returned before its data write began: {result:?}"), + result = entered_rx => result.context("destination write never began")?, + } + + assert!( + tokio::time::timeout(Duration::from_millis(25), &mut flush) + .await + .is_err(), + "final flush returned while the destination write was still blocked" + ); + assert!(!write_completed.load(Ordering::SeqCst)); + + release_tx + .send(()) + .map_err(|_| anyhow::anyhow!("destination write stopped before gate release"))?; + flush + .await + .context("snapshot buffer stopped during final flush")??; + assert!(write_completed.load(Ordering::SeqCst)); + Ok(()) + } + #[actix::test] async fn test_shutdown_force_flushes_open_batches() -> Result<()> { // An open, debounced batch whose timelock has NOT fired must still be diff --git a/crates/events/src/snapshot_buffer/timelock_queue.rs b/crates/events/src/snapshot_buffer/timelock_queue.rs index ba942ba5d8..920a49c202 100644 --- a/crates/events/src/snapshot_buffer/timelock_queue.rs +++ b/crates/events/src/snapshot_buffer/timelock_queue.rs @@ -3,6 +3,8 @@ // This file is provided WITHOUT ANY WARRANTY; // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +#[cfg(test)] +use crate::AggregateId; use actix::{Actor, Addr, AsyncContext, Handler, Message, Recipient}; use e3_utils::MAILBOX_LIMIT; use std::{ @@ -13,23 +15,24 @@ use std::{ }; use tracing::debug; -use super::batch_router::FlushSeq; +use super::batch_router::{FlushSeq, SnapshotKey}; #[derive(Message)] #[rtype(result = "()")] pub struct StartTimelock { - seq: u64, + key: SnapshotKey, now: Duration, delay: Duration, } impl StartTimelock { - pub fn new(seq: u64, now: Duration, delay: Duration) -> Self { - Self { seq, now, delay } + pub(super) fn new(key: SnapshotKey, now: Duration, delay: Duration) -> Self { + Self { key, now, delay } } - pub fn new_micros(seq: u64, now: u64, delay: u64) -> Self { + #[cfg(test)] + fn new_micros(seq: u64, now: u64, delay: u64) -> Self { Self::new( - seq, + SnapshotKey::new(AggregateId::new(0), seq), Duration::from_micros(now), Duration::from_micros(delay), ) @@ -60,18 +63,20 @@ pub struct Tick; #[derive(Debug, Clone, Copy, Eq, PartialEq)] pub struct Timelock { expiry: Duration, - seq: u64, + key: SnapshotKey, } impl Timelock { - pub fn new(expiry: Duration, seq: u64) -> Self { - Self { expiry, seq } + pub fn new(expiry: Duration, key: SnapshotKey) -> Self { + Self { expiry, key } } } impl Ord for Timelock { fn cmp(&self, other: &Self) -> Ordering { - self.expiry.cmp(&other.expiry) + self.expiry + .cmp(&other.expiry) + .then_with(|| self.key.cmp(&other.key)) } } @@ -142,7 +147,7 @@ impl Handler for TimelockQueue { fn handle(&mut self, msg: StartTimelock, _: &mut Self::Context) -> Self::Result { debug!("Start timelock: {:?}", msg.delay); let expiry = msg.now + msg.delay; - self.timelocks.push(Reverse(Timelock::new(expiry, msg.seq))); + self.timelocks.push(Reverse(Timelock::new(expiry, msg.key))); } } @@ -160,9 +165,13 @@ impl Handler for TimelockQueue { while !self.timelocks.is_empty() && self.next_timelock_lt(now_time) { if let Some(tl) = self.timelocks.pop() { - let seq = tl.0.seq; - debug!("Flushing seq {}", seq); - self.batch_router.do_send(FlushSeq(seq)); + let key = tl.0.key; + debug!( + aggregate = %key.aggregate_id(), + seq = key.seq(), + "Flushing expired snapshot batch" + ); + self.batch_router.do_send(FlushSeq(key)); } } } @@ -239,7 +248,7 @@ mod tests { type Result = (); fn handle(&mut self, msg: FlushSeq, _: &mut Self::Context) -> Self::Result { - self.received_seqs.lock().unwrap().push(msg.0); + self.received_seqs.lock().unwrap().push(msg.seq()); } } From a6fb687da6b35569fb8ea63cc9cd215ac082befc Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 15:27:25 +0500 Subject: [PATCH 23/62] fix(sync): restore full logical clock state --- crates/events/src/bus_handle.rs | 32 +++++++++++++----- crates/events/src/hlc.rs | 56 +++++++++++++++++++++----------- crates/events/src/hlc_factory.rs | 19 ++++++----- crates/sync/src/actors/sync.rs | 46 +++++++++++++++++++++----- 4 files changed, 109 insertions(+), 44 deletions(-) diff --git a/crates/events/src/bus_handle.rs b/crates/events/src/bus_handle.rs index 860fc242cd..efe6df60ad 100644 --- a/crates/events/src/bus_handle.rs +++ b/crates/events/src/bus_handle.rs @@ -190,13 +190,10 @@ impl BusHandle { Ok(ts.into()) } - /// Seed the HLC physical-time floor from a persisted, packed HLC timestamp - /// so freshly-`tick`ed events after a restart never sort before durable - /// history (H15). Pass the highest timestamp recovered from persisted - /// aggregate state; only the physical-time component is used. - pub fn seed_clock(&self, packed_ts: u128) { - let ts = HlcTimestamp::from(packed_ts); - self.hlc.seed_physical_floor(ts.ts); + /// Restore the HLC ordering floor from the greatest persisted packed timestamp. + pub fn seed_clock(&self, packed_ts: u128) -> Result<()> { + self.hlc.seed_from_history(HlcTimestamp::from(packed_ts))?; + Ok(()) } /// Pipe events from this handle to the other handle only when the predicate returns true @@ -486,8 +483,9 @@ mod tests { use e3_ciphernode_builder::EventSystem; // NOTE: We cannot pull from crate as the features will be missing as they are not default. use e3_events::{ - hlc::Hlc, prelude::*, BusHandle, EventPublisher, EventType, InterfoldEvent, - InterfoldEventData, TestEvent, + hlc::{Hlc, HlcTimestamp}, + prelude::*, + BusHandle, EventPublisher, EventType, InterfoldEvent, InterfoldEventData, TestEvent, }; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use tokio::time::sleep; @@ -499,6 +497,22 @@ mod tests { .as_micros() as u64 } + #[actix::test] + async fn restart_seed_keeps_next_event_after_durable_logical_time() -> anyhow::Result<()> { + let bus = EventSystem::new() + .with_fresh_bus() + .handle()? + .enable_with_hlc(Hlc::new(7).with_clock(|| 1_000)); + let persisted = HlcTimestamp::new(5_000, 17, 99); + + bus.seed_clock(persisted.to_u128())?; + let next = HlcTimestamp::from(bus.ts()?); + + assert_eq!(next, HlcTimestamp::new(5_000, 18, 7)); + assert!(next > persisted); + Ok(()) + } + #[actix::test] async fn test_hlc_events() -> anyhow::Result<()> { #[derive(Message)] diff --git a/crates/events/src/hlc.rs b/crates/events/src/hlc.rs index 7c306dc882..29511496e1 100644 --- a/crates/events/src/hlc.rs +++ b/crates/events/src/hlc.rs @@ -249,20 +249,17 @@ impl Hlc { self.node } - /// Seed the clock's physical-time floor so it can never regress below an - /// already-emitted timestamp across a restart (H15). A node boots with a - /// fresh HLC at `ts: 0` and re-derives time from the system clock, so a - /// backward wall-clock jump would let freshly-`tick`ed events sort *before* - /// events already persisted before the crash. Seeding `inner.ts` to the - /// highest persisted timestamp makes `tick`/`receive` (which both take a - /// `max`) monotone with respect to durable history. Counter is reset to 0: - /// the very next `tick` at the same `ts` bumps it to 1, staying above the - /// persisted value. - pub fn seed_physical_floor(&self, ts: u64) { + /// Restore the ordering floor from the greatest timestamp in durable history. + /// + /// The persisted node identifier is deliberately not adopted: it identifies the process + /// that created the historical event, while this clock must retain the current node's + /// identity. Restoring both the physical time and logical counter guarantees the next tick is + /// strictly greater than the persisted timestamp even when wall time has moved backwards. + pub fn seed_from_history(&self, persisted: HlcTimestamp) { let mut inner = self.inner.lock().unwrap(); - if ts > inner.ts { - inner.ts = ts; - inner.counter = 0; + if (persisted.ts, persisted.counter) > (inner.ts, inner.counter) { + inner.ts = persisted.ts; + inner.counter = persisted.counter; } } @@ -660,27 +657,48 @@ mod tests { } #[test] - fn seed_physical_floor_prevents_regression_below_seed() { + fn seed_from_history_preserves_logical_counter() { // Simulate a restart where the wall clock jumped backwards: the system // clock reports an earlier physical time than already-emitted history. let hlc = Hlc::new(1).with_clock(|| 1_000); - hlc.seed_physical_floor(5_000); + let persisted = HlcTimestamp::new(5_000, 17, 99); + hlc.seed_from_history(persisted); let ts = hlc.tick().unwrap(); - // Even though now_physical() == 1_000, the seed floor keeps us ahead. + // Even though now_physical() == 1_000, the full seed keeps us ahead. assert_eq!(ts.ts, 5_000); - assert_eq!(ts.counter, 1); + assert_eq!(ts.counter, 18); + assert!(ts > persisted); + } + + #[test] + fn seed_from_history_uses_counter_to_choose_same_physical_time() { + let hlc = Hlc::with_state(5_000, 4, 1).with_clock(|| 1_000); + hlc.seed_from_history(HlcTimestamp::new(5_000, 17, 99)); + let ts = hlc.tick().unwrap(); + assert_eq!(ts, HlcTimestamp::new(5_000, 18, 1)); } #[test] - fn seed_physical_floor_is_noop_when_below_current() { + fn seed_from_history_is_noop_when_below_current() { let hlc = Hlc::with_state(9_000, 4, 1); - hlc.seed_physical_floor(5_000); + hlc.seed_from_history(HlcTimestamp::new(5_000, u32::MAX, 99)); let ts = hlc.get(); assert_eq!(ts.ts, 9_000); assert_eq!(ts.counter, 4); } + #[test] + fn seed_from_history_handles_logical_counter_overflow() { + let hlc = Hlc::new(1).with_clock(|| 1_000); + let persisted = HlcTimestamp::new(5_000, u32::MAX, 99); + hlc.seed_from_history(persisted); + + let ts = hlc.tick().unwrap(); + assert_eq!(ts, HlcTimestamp::new(5_001, 0, 1)); + assert!(ts > persisted); + } + #[test] fn receive_picks_max_timestamp() { let hlc = Hlc::with_state(100, 5, 1).with_max_drift(u64::MAX); diff --git a/crates/events/src/hlc_factory.rs b/crates/events/src/hlc_factory.rs index 00f7dd7c49..2c3add463f 100644 --- a/crates/events/src/hlc_factory.rs +++ b/crates/events/src/hlc_factory.rs @@ -96,14 +96,17 @@ impl HlcFactory { } } - /// Seed the underlying clock's physical-time floor (H15). No-op until the - /// HLC has been enabled. See `Hlc::seed_physical_floor`. - pub fn seed_physical_floor(&self, ts: u64) { - if let Ok(guard) = self.hlc.lock() { - if let HlcState::Ready(hlc) = &*guard { - hlc.seed_physical_floor(ts); - } - } + /// Restore the underlying clock's durable ordering floor. + /// + /// Startup must not silently continue if the clock is unavailable: doing so can create event + /// timestamps behind already-persisted history after a wall-clock rollback. + pub fn seed_from_history(&self, timestamp: HlcTimestamp) -> Result<(), HlcFactoryError> { + let guard = self.hlc.lock()?; + let HlcState::Ready(hlc) = &*guard else { + return Err(HlcFactoryError::NotReady); + }; + hlc.seed_from_history(timestamp); + Ok(()) } } diff --git a/crates/sync/src/actors/sync.rs b/crates/sync/src/actors/sync.rs index 0aea2fde67..04468c2408 100644 --- a/crates/sync/src/actors/sync.rs +++ b/crates/sync/src/actors/sync.rs @@ -13,10 +13,10 @@ use actix::{Message, Recipient}; use anyhow::{bail, Result}; use e3_data::Repositories; use e3_events::{ - AggregateConfig, BusHandle, CorrelationId, EffectsEnabled, Event, EventPublisher, - EventStoreQueryBy, EventStoreQueryResponse, EventSubscriber, EventType, EvmEventConfig, - HistoricalEvmEventsReceived, HistoricalEvmSyncStart, HistoricalNetSyncStart, InterfoldEvent, - InterfoldEventData, SeqAgg, SyncEnded, Unsequenced, + AggregateConfig, BusHandle, CorrelationId, EffectsEnabled, Event, EventContextAccessors, + EventPublisher, EventStoreQueryBy, EventStoreQueryResponse, EventSubscriber, EventType, + EvmEventConfig, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, HistoricalNetSyncStart, + InterfoldEvent, InterfoldEventData, SeqAgg, SyncEnded, Unsequenced, }; use e3_utils::actix::channel as actix_toolbox; use std::time::Duration; @@ -48,11 +48,11 @@ pub async fn sync( snapshot.aggregates().len() ); - // 1b. Seed the HLC physical-time floor from the highest persisted aggregate - // timestamp so events created after this restart never sort before - // durable history, even if the wall clock jumped backwards (H15). + // 1b. Restore the HLC ordering floor from the highest persisted aggregate + // timestamp so events created after this restart remain strictly after + // durable history, including its logical counter, even if wall time moved backwards. if let Some(max_ts) = snapshot.to_net_config().values().copied().max() { - bus.seed_clock(max_ts); + bus.seed_clock(max_ts)?; } // 2. Determine the evm blocks to read from based on the SnapshotMeta @@ -69,6 +69,7 @@ pub async fn sync( ))?; let events = rx.await?.into_events(); info!("{} EventStore events loaded.", events.len()); + seed_clock_from_replay(bus, &events)?; info!("Replaying events to actors..."); // 4. Replay the EventStore events to all listeners (except effects). @@ -174,6 +175,15 @@ pub async fn sync( Ok(()) } +fn seed_clock_from_replay(bus: &BusHandle, events: &[InterfoldEvent]) -> Result<()> { + // Snapshot metadata can lag the append-only log after a failed snapshot write. Seed from the + // actual replay set before any subscriber can emit follow-up work, otherwise new local events + // may receive timestamps behind durable post-snapshot history. + if let Some(max_ts) = events.iter().map(EventContextAccessors::ts).max() { + bus.seed_clock(max_ts)?; + } + Ok(()) +} /// Verify the on-disk schema version against this binary and either stamp a /// fresh marker (first boot) or halt loudly on an incompatible upgrade or /// downgrade (H19/H20). Uses a synchronous write so the marker is durable @@ -253,9 +263,11 @@ impl SnapshotLoaded { #[cfg(test)] mod tests { + use super::seed_clock_from_replay; use crate::domain::SyncPlanner; use e3_ciphernode_builder::EventSystem; use e3_events::{ + hlc::{Hlc, HlcTimestamp}, EffectsEnabled, Event, EventPublisher, EventSubscriber, EventType, EvmEventConfig, HistoricalEvmSyncStart, InterfoldEvent, InterfoldEventData, SyncEnded, TakeEvents, Unsequenced, @@ -336,6 +348,24 @@ mod tests { Ok(()) } + #[actix::test] + async fn replay_seeds_clock_from_post_snapshot_log_history() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system + .handle()? + .enable_with_hlc(Hlc::new(7).with_clock(|| 1_000)); + let durable = HlcTimestamp::new(5_000, 17, 99); + let events = vec![InterfoldEvent::::test_event("durable") + .id(1) + .ts(durable.to_u128()) + .seq(1) + .build()]; + + seed_clock_from_replay(&bus, &events)?; + + assert!(HlcTimestamp::from(bus.ts()?) > durable); + Ok(()) + } /// Verify that `run_once::` correctly gates event subscriptions. /// /// Simulates the sync flow: From b4126aef2f66ffa6396ecdfdb0a704f93f580abf Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 15:29:41 +0500 Subject: [PATCH 24/62] fix(sync): reject incomplete EVM history --- crates/sync/src/actors/sync.rs | 69 ++++++++++++++++--- .../src/domain/historical_evm_collector.rs | 8 ++- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/crates/sync/src/actors/sync.rs b/crates/sync/src/actors/sync.rs index 04468c2408..a228e72a32 100644 --- a/crates/sync/src/actors/sync.rs +++ b/crates/sync/src/actors/sync.rs @@ -21,7 +21,7 @@ use e3_events::{ use e3_utils::actix::channel as actix_toolbox; use std::time::Duration; use tokio::sync::mpsc::Receiver; -use tracing::{info, warn}; +use tracing::info; pub async fn sync( bus: &BusHandle, @@ -118,7 +118,7 @@ pub async fn sync( info!("Loading historical blockchain events..."); let (addr, rx) = actix_toolbox::mpsc::(256); bus.publish_without_context(HistoricalEvmSyncStart::new(addr, evm_config.clone()))?; - let historical_evm_events = collect_historical_evm_events(rx, &evm_config).await; + let historical_evm_events = collect_historical_evm_events(rx, &evm_config).await?; info!( "{} historical blockchain events loaded.", historical_evm_events.len() @@ -207,7 +207,7 @@ async fn check_schema_version(repositories: &Repositories) -> Result<()> { pub async fn collect_historical_evm_events( mut receiver: Receiver, config: &EvmEventConfig, -) -> Vec> { +) -> Result>> { let mut collector = HistoricalEvmCollector::new(config); let progress_interval = Duration::from_secs(30); @@ -227,9 +227,8 @@ pub async fn collect_historical_evm_events( } } Ok(None) => { - // Channel closed — sender dropped - warn!("Historical events channel closed before all chains reported"); - break; + let remaining = collector.remaining(); + bail!("historical EVM event channel closed before chains reported: {remaining:?}"); } Err(_) => { // Not a failure — just a progress heartbeat @@ -243,7 +242,7 @@ pub async fn collect_historical_evm_events( } } - collector.into_events() + Ok(collector.into_events()) } #[derive(Message)] @@ -263,15 +262,16 @@ impl SnapshotLoaded { #[cfg(test)] mod tests { - use super::seed_clock_from_replay; + use super::{collect_historical_evm_events, seed_clock_from_replay}; use crate::domain::SyncPlanner; use e3_ciphernode_builder::EventSystem; use e3_events::{ hlc::{Hlc, HlcTimestamp}, EffectsEnabled, Event, EventPublisher, EventSubscriber, EventType, EvmEventConfig, - HistoricalEvmSyncStart, InterfoldEvent, InterfoldEventData, SyncEnded, TakeEvents, - Unsequenced, + EvmEventConfigChain, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, InterfoldEvent, + InterfoldEventData, SyncEnded, TakeEvents, Unsequenced, }; + use std::collections::BTreeMap; fn make_historical_evm_sync_start() -> HistoricalEvmSyncStart { HistoricalEvmSyncStart { @@ -280,6 +280,55 @@ mod tests { } } + fn evm_config(chains: &[u64]) -> EvmEventConfig { + EvmEventConfig::from_config( + chains + .iter() + .map(|chain_id| (*chain_id, EvmEventConfigChain::new(0))) + .collect::>(), + ) + } + + fn historical_batch(chain_id: u64, event_count: usize) -> HistoricalEvmEventsReceived { + let events = (0..event_count) + .map(|index| { + InterfoldEvent::::test_event("historical") + .id(index as u64 + 1) + .build() + }) + .collect(); + HistoricalEvmEventsReceived::new(events, chain_id) + } + + #[actix::test] + async fn historical_evm_collection_fails_when_any_chain_disconnects() { + let (sender, receiver) = tokio::sync::mpsc::channel(2); + sender.send(historical_batch(1, 2)).await.unwrap(); + drop(sender); + + let error = collect_historical_evm_events(receiver, &evm_config(&[1, 2, 3])) + .await + .unwrap_err(); + + assert_eq!( + error.to_string(), + "historical EVM event channel closed before chains reported: [2, 3]" + ); + } + + #[actix::test] + async fn historical_evm_collection_returns_only_after_every_chain_reports() { + let (sender, receiver) = tokio::sync::mpsc::channel(2); + sender.send(historical_batch(2, 3)).await.unwrap(); + sender.send(historical_batch(1, 2)).await.unwrap(); + + let events = collect_historical_evm_events(receiver, &evm_config(&[1, 2])) + .await + .unwrap(); + + assert_eq!(events.len(), 5); + } + #[actix::test] async fn infrastructure_events_are_filtered_during_replay() -> anyhow::Result<()> { let system = EventSystem::new().with_fresh_bus(); diff --git a/crates/sync/src/domain/historical_evm_collector.rs b/crates/sync/src/domain/historical_evm_collector.rs index 001d0b53dd..ae1f98c179 100644 --- a/crates/sync/src/domain/historical_evm_collector.rs +++ b/crates/sync/src/domain/historical_evm_collector.rs @@ -59,7 +59,13 @@ impl HistoricalEvmCollector { /// Chains still outstanding (expected but not yet received). pub fn remaining(&self) -> Vec { - self.expected.difference(&self.received).copied().collect() + let mut remaining = self + .expected + .difference(&self.received) + .copied() + .collect::>(); + remaining.sort_unstable(); + remaining } /// Consume the collector and return the accumulated events. From fae83d0fa613e223149760ded2b855299493f8ff Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 16:18:35 +0500 Subject: [PATCH 25/62] fix(sync): reject unversioned persisted state --- .../src/ciphernode_builder.rs | 8 +- crates/data/src/data_store.rs | 15 ++++ crates/data/src/in_mem.rs | 10 ++- crates/data/src/in_mem_kv_store.rs | 6 ++ crates/data/src/sled_db.rs | 4 + crates/data/src/sled_store.rs | 11 ++- crates/entrypoint/src/validate.rs | 17 ++-- crates/sync/src/actors/sync.rs | 89 +++++++++++++++++-- crates/sync/src/domain/schema_version.rs | 33 +++++-- 9 files changed, 163 insertions(+), 30 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 3dd8fb598e..5807a354e3 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -42,7 +42,7 @@ use e3_sortition::{ FinalizedCommitteesRepositoryFactory, NodeStateRepositoryFactory, Sortition, SortitionBackend, SortitionRepositoryFactory, }; -use e3_sync::sync; +use e3_sync::{preflight_schema_version, sync}; use e3_utils::SharedRng; use e3_zk_prover::{setup_zk_actors, ZkBackend}; use libp2p::PeerId; @@ -532,6 +532,12 @@ impl CiphernodeBuilder { let store = event_system.store()?; let eventstore = event_system.eventstore_reader()?; let repositories = Arc::new(store.repositories()); + + // Establish storage compatibility before signers, actors, or forked runtime events can + // create durable state. Running this only inside `sync` is too late: actor startup can + // make a fresh store non-empty and cause it to look like unversioned legacy data. + preflight_schema_version(&repositories, &aggregate_config, &eventstore.seq()).await?; + let mut provider_cache = provider_cache.with_write_support(Arc::clone(&self.cipher), Arc::clone(&repositories)); diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index 1bc61d89f2..75c190306e 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -33,6 +33,10 @@ pub enum StoreAddr { #[rtype(result = "Result<()>")] pub struct ShutdownStore; +#[derive(Message, Debug)] +#[rtype(result = "Result")] +pub(crate) struct StoreIsEmpty; + impl StoreAddr { pub fn to_maybe_in_mem(&self) -> Option<&Addr> { match self { @@ -60,6 +64,17 @@ pub struct DataStore { } impl DataStore { + /// Return whether the complete backing key/value store contains no entries. + /// + /// This intentionally ignores the current scope: schema compatibility is a property of the + /// whole physical store, not one repository prefix. + pub async fn is_empty(&self) -> Result { + match &self.addr { + StoreAddr::InMem(store) => Ok(store.send(StoreIsEmpty).await??), + StoreAddr::Sled(store) => Ok(store.send(StoreIsEmpty).await??), + } + } + /// Read data at the scope location pub async fn read(&self) -> Result> where diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index e661159cbe..e398e1e025 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -6,7 +6,7 @@ use crate::{ in_mem_kv_store::{DataOp, InMemKvStore}, - ShutdownStore, + ShutdownStore, StoreIsEmpty, }; use actix::{Actor, ActorContext, Handler, Message}; use anyhow::Result; @@ -75,6 +75,14 @@ impl Handler for InMemStore { } } +impl Handler for InMemStore { + type Result = Result; + + fn handle(&mut self, _: StoreIsEmpty, _: &mut Self::Context) -> Self::Result { + Ok(self.store.is_empty()) + } +} + impl Handler for InMemStore { type Result = Result; diff --git a/crates/data/src/in_mem_kv_store.rs b/crates/data/src/in_mem_kv_store.rs index 12c2dac271..b9eeccf7ab 100644 --- a/crates/data/src/in_mem_kv_store.rs +++ b/crates/data/src/in_mem_kv_store.rs @@ -69,6 +69,10 @@ impl InMemKvStore { self.db.get(&key.to_vec()).cloned() } + pub fn is_empty(&self) -> bool { + self.db.entries().is_empty() + } + /// Returns the captured operation log. pub fn log(&self) -> Vec { self.log.clone() @@ -103,8 +107,10 @@ mod tests { #[test] fn insert_get_remove() { let mut store = InMemKvStore::new(false); + assert!(store.is_empty()); store.insert(b"a".to_vec(), b"1".to_vec(), None); store.insert(b"b".to_vec(), b"2".to_vec(), None); + assert!(!store.is_empty()); assert_eq!(Some(b"1".to_vec()), store.get(b"a")); assert_eq!(Some(b"2".to_vec()), store.get(b"b")); assert_eq!(None, store.get(b"missing")); diff --git a/crates/data/src/sled_db.rs b/crates/data/src/sled_db.rs index 17b3541539..e3a0562182 100644 --- a/crates/data/src/sled_db.rs +++ b/crates/data/src/sled_db.rs @@ -87,6 +87,10 @@ impl SledDb { Ok(res.map(|v| v.to_vec())) } + pub fn is_empty(&self) -> bool { + self.db.is_empty() + } + pub fn flush(&self) -> Result<()> { self.db.flush()?; Ok(()) diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index aa18ec4b61..9099320ac0 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{ShutdownStore, SledDb}; +use crate::{ShutdownStore, SledDb, StoreIsEmpty}; use actix::{Actor, ActorContext, Addr, Handler, ResponseFuture}; use anyhow::{Context, Result}; use e3_events::{BusHandle, EType, ErrorDispatcher, Flush, InterfoldEvent, Unsequenced}; @@ -147,6 +147,15 @@ impl Handler for SledStore { } } +impl Handler for SledStore { + type Result = Result; + + fn handle(&mut self, _: StoreIsEmpty, _: &mut Self::Context) -> Self::Result { + let db = self.db.as_ref().context("SledStore is closed")?; + Ok(db.is_empty()) + } +} + impl Handler for SledStore { type Result = Result<()>; fn handle(&mut self, _: Flush, _: &mut Self::Context) -> Self::Result { diff --git a/crates/entrypoint/src/validate.rs b/crates/entrypoint/src/validate.rs index 69b176bc11..bc712ef39b 100644 --- a/crates/entrypoint/src/validate.rs +++ b/crates/entrypoint/src/validate.rs @@ -210,9 +210,10 @@ pub async fn validate_node(config: &AppConfig) -> Result { // framing and decode checks. Cross-check each persisted replay cursor. let repositories = get_repositories(config)?; let persisted_schema = repositories.schema_version().read().await?; + let has_existing_state = total_events > 0 || !repositories.store.is_empty().await?; report.push(check_schema_compatibility( persisted_schema, - total_events > 0, + has_existing_state, )); for (agg, events) in &events_by_aggregate { let seqs: Vec = events.iter().map(|e| e.seq()).collect(); @@ -237,25 +238,17 @@ pub async fn validate_node(config: &AppConfig) -> Result { /// Verify that this binary can safely interpret the persisted schema. A missing /// marker is acceptable only for a genuinely empty store; stamping a version on /// non-empty legacy bytes would assert compatibility without evidence. -fn check_schema_compatibility(persisted: Option, has_events: bool) -> CheckResult { +fn check_schema_compatibility(persisted: Option, has_existing_state: bool) -> CheckResult { let name = "schema"; - match decide_schema_version(persisted, SCHEMA_VERSION) { + match decide_schema_version(persisted, SCHEMA_VERSION, has_existing_state) { SchemaVersionDecision::Proceed => CheckResult::pass( name, format!("on-disk schema version {SCHEMA_VERSION} matches this binary"), ), - SchemaVersionDecision::WriteCurrent if !has_events => CheckResult::pass( + SchemaVersionDecision::WriteCurrent => CheckResult::pass( name, format!("empty store will be initialized at schema version {SCHEMA_VERSION}"), ), - SchemaVersionDecision::WriteCurrent => CheckResult::fail( - name, - format!( - "non-empty event log has no schema marker; compatibility with schema version \ - {SCHEMA_VERSION} cannot be proven. Back up the stopped node and use an explicit \ - migration or controlled resync" - ), - ), SchemaVersionDecision::Halt(reason) => CheckResult::fail(name, reason), } } diff --git a/crates/sync/src/actors/sync.rs b/crates/sync/src/actors/sync.rs index a228e72a32..bfc32997d5 100644 --- a/crates/sync/src/actors/sync.rs +++ b/crates/sync/src/actors/sync.rs @@ -10,7 +10,7 @@ use crate::domain::{ }; use crate::SyncRepositoryFactory; use actix::{Message, Recipient}; -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; use e3_data::Repositories; use e3_events::{ AggregateConfig, BusHandle, CorrelationId, EffectsEnabled, Event, EventContextAccessors, @@ -36,7 +36,7 @@ pub async fn sync( // 0b. Verify the on-disk schema version is compatible with this binary // before touching any persisted state, so an incompatible upgrade or // downgrade halts loudly instead of silently loading garbage (H19/H20). - check_schema_version(repositories).await?; + preflight_schema_version(repositories, aggregate_config, eventstore).await?; // 1. Load snapsshot metadata info!("Loading snapshot metadata..."); @@ -184,14 +184,25 @@ fn seed_clock_from_replay(bus: &BusHandle, events: &[InterfoldEvent]) -> Result< } Ok(()) } -/// Verify the on-disk schema version against this binary and either stamp a -/// fresh marker (first boot) or halt loudly on an incompatible upgrade or -/// downgrade (H19/H20). Uses a synchronous write so the marker is durable -/// before any further state is loaded. -async fn check_schema_version(repositories: &Repositories) -> Result<()> { +/// Validate or initialize the durable schema marker before runtime actors can write state. +/// +/// A fresh empty store is stamped synchronously; incompatible or unversioned non-empty state +/// halts. The check is idempotent so composition roots may run it before actor construction while +/// [`sync`] retains the same guard for direct callers. +pub async fn preflight_schema_version( + repositories: &Repositories, + aggregate_config: &AggregateConfig, + eventstore: &Recipient>, +) -> Result<()> { let repo = repositories.schema_version(); let persisted = repo.read().await?; - match decide_schema_version(persisted, SCHEMA_VERSION) { + let has_existing_state = if persisted.is_none() { + !repositories.store.is_empty().await? + || event_logs_have_events(aggregate_config, eventstore).await? + } else { + false + }; + match decide_schema_version(persisted, SCHEMA_VERSION, has_existing_state) { SchemaVersionDecision::Proceed => Ok(()), SchemaVersionDecision::WriteCurrent => { info!("Stamping on-disk schema version {SCHEMA_VERSION}."); @@ -204,6 +215,27 @@ async fn check_schema_version(repositories: &Repositories) -> Result<()> { } } +async fn event_logs_have_events( + aggregate_config: &AggregateConfig, + eventstore: &Recipient>, +) -> Result { + let query = aggregate_config + .aggregates() + .into_iter() + .map(|aggregate_id| (aggregate_id, 1)) + .collect(); + let (response, receiver) = actix_toolbox::oneshot::(); + eventstore + .send(EventStoreQueryBy::::new(CorrelationId::new(), query, response).with_limit(1)) + .await + .context("event-store router stopped during schema preflight")?; + Ok(!receiver + .await + .context("event-store query stopped during schema preflight")? + .into_events() + .is_empty()) +} + pub async fn collect_historical_evm_events( mut receiver: Receiver, config: &EvmEventConfig, @@ -262,9 +294,12 @@ impl SnapshotLoaded { #[cfg(test)] mod tests { - use super::{collect_historical_evm_events, seed_clock_from_replay}; + use super::{ + collect_historical_evm_events, preflight_schema_version, seed_clock_from_replay, + }; use crate::domain::SyncPlanner; use e3_ciphernode_builder::EventSystem; + use e3_data::Repositories; use e3_events::{ hlc::{Hlc, HlcTimestamp}, EffectsEnabled, Event, EventPublisher, EventSubscriber, EventType, EvmEventConfig, @@ -300,6 +335,42 @@ mod tests { HistoricalEvmEventsReceived::new(events, chain_id) } + #[actix::test] + async fn schema_preflight_rejects_unversioned_snapshot_state() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let store = system.store()?; + store.scope("legacy").write_sync(7_u64).await?; + let repositories = Repositories::from(&store); + let eventstore = system.eventstore_reader()?.seq(); + + let error = + preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) + .await + .unwrap_err(); + + assert!(error.to_string().contains("no schema marker")); + Ok(()) + } + + #[actix::test] + async fn schema_preflight_rejects_unversioned_event_log() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let store = system.store()?; + let repositories = Repositories::from(&store); + let bus = system.handle()?.enable("unversioned-event-log"); + bus.publish_without_context(e3_events::TestEvent::new("legacy", 1))?; + bus.flush_event_pipeline().await?; + let eventstore = system.eventstore_reader()?.seq(); + + let error = + preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) + .await + .unwrap_err(); + + assert!(error.to_string().contains("no schema marker")); + Ok(()) + } + #[actix::test] async fn historical_evm_collection_fails_when_any_chain_disconnects() { let (sender, receiver) = tokio::sync::mpsc::channel(2); diff --git a/crates/sync/src/domain/schema_version.rs b/crates/sync/src/domain/schema_version.rs index ffd2d77180..ce0c68cafc 100644 --- a/crates/sync/src/domain/schema_version.rs +++ b/crates/sync/src/domain/schema_version.rs @@ -33,9 +33,18 @@ pub enum SchemaVersionDecision { /// additive evolution and removed the previous `#[serde(default)]` shims, any /// version mismatch implies a breaking change with no migration path, so both /// older on-disk data (upgrade) and newer on-disk data (downgrade) halt. -pub fn decide_schema_version(persisted: Option, current: u32) -> SchemaVersionDecision { +pub fn decide_schema_version( + persisted: Option, + current: u32, + has_existing_state: bool, +) -> SchemaVersionDecision { match persisted { - None => SchemaVersionDecision::WriteCurrent, + None if !has_existing_state => SchemaVersionDecision::WriteCurrent, + None => SchemaVersionDecision::Halt(format!( + "On-disk state has no schema marker, so compatibility with schema version {current} \ + cannot be proven. Halting; back up the stopped node and use an explicit migration or \ + controlled resync." + )), Some(v) if v == current => SchemaVersionDecision::Proceed, Some(v) if v > current => SchemaVersionDecision::Halt(format!( "On-disk schema version {v} is newer than this binary's supported version {current}. \ @@ -57,7 +66,7 @@ mod tests { #[test] fn fresh_store_writes_current() { assert_eq!( - decide_schema_version(None, 3), + decide_schema_version(None, 3, false), SchemaVersionDecision::WriteCurrent ); } @@ -65,14 +74,14 @@ mod tests { #[test] fn exact_match_proceeds() { assert_eq!( - decide_schema_version(Some(3), 3), + decide_schema_version(Some(3), 3, true), SchemaVersionDecision::Proceed ); } #[test] fn older_on_disk_halts_as_upgrade() { - let d = decide_schema_version(Some(2), 3); + let d = decide_schema_version(Some(2), 3, true); match d { SchemaVersionDecision::Halt(msg) => { assert!(msg.contains("older")); @@ -84,7 +93,7 @@ mod tests { #[test] fn newer_on_disk_halts_as_downgrade() { - let d = decide_schema_version(Some(4), 3); + let d = decide_schema_version(Some(4), 3, true); match d { SchemaVersionDecision::Halt(msg) => { assert!(msg.contains("newer")); @@ -93,4 +102,16 @@ mod tests { other => panic!("expected Halt, got {other:?}"), } } + + #[test] + fn missing_marker_on_existing_state_halts() { + let d = decide_schema_version(None, 3, true); + match d { + SchemaVersionDecision::Halt(message) => { + assert!(message.contains("no schema marker")); + assert!(message.contains("controlled resync")); + } + other => panic!("expected Halt, got {other:?}"), + } + } } From d0f5f1dba217f45c7e41f43d65dfa440e110618a Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 16:31:33 +0500 Subject: [PATCH 26/62] fix(serialization): bound external bincode decoding --- Cargo.lock | 1 + crates/data/src/commit_log_event_log.rs | 7 +- crates/data/src/data_store.rs | 2 +- crates/data/src/in_mem_kv_store.rs | 2 +- crates/data/src/persistable.rs | 2 +- crates/events/src/interfold_event/mod.rs | 26 +++++++- crates/net/src/actors/net_sync_manager.rs | 3 +- crates/net/src/domain/event_conversion.rs | 4 +- crates/net/src/domain/mod.rs | 1 + crates/net/src/domain/net_event_batch.rs | 5 +- crates/net/src/domain/wire.rs | 18 ++++++ crates/net/src/events.rs | 28 +++++++- crates/net/src/net_interface.rs | 12 ++-- crates/utils/Cargo.toml | 1 + crates/utils/src/bounded_bincode.rs | 78 +++++++++++++++++++++++ crates/utils/src/lib.rs | 2 + 16 files changed, 172 insertions(+), 20 deletions(-) create mode 100644 crates/net/src/domain/wire.rs create mode 100644 crates/utils/src/bounded_bincode.rs diff --git a/Cargo.lock b/Cargo.lock index cd3bafa82a..d1d8ec1745 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3924,6 +3924,7 @@ dependencies = [ "actix", "alloy", "anyhow", + "bincode 1.3.3", "derivative", "e3-utils-derive", "hex", diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 1ad92ccf19..8341ddb325 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -77,12 +77,13 @@ impl CommitLogEventLog { let mut count = 0; for msg in message_buf.iter() { let seq = msg.offset() + 1; - let event = bincode::deserialize::>(msg.payload()) - .with_context(|| { + let event = InterfoldEvent::::from_bytes(msg.payload()).with_context( + || { format!( "commit log event at sequence {seq} failed to decode; log is corrupt" ) - })?; + }, + )?; events.push((seq, event)); current_offset = msg.offset() + 1; count += 1; diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index 75c190306e..4dab90a245 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -88,7 +88,7 @@ impl DataStore { if bytes == [0] { return Ok(None); } - Ok(Some(bincode::deserialize(&bytes)?)) + Ok(Some(e3_utils::deserialize_exact(&bytes)?)) } /// Writes data to the scope location diff --git a/crates/data/src/in_mem_kv_store.rs b/crates/data/src/in_mem_kv_store.rs index b9eeccf7ab..144009eeff 100644 --- a/crates/data/src/in_mem_kv_store.rs +++ b/crates/data/src/in_mem_kv_store.rs @@ -87,7 +87,7 @@ impl InMemKvStore { /// Reconstructs a store from a bincode `BTreeMap` dump. pub fn from_dump(bytes: &[u8], capture: bool) -> Result { let map: BTreeMap, Vec> = - bincode::deserialize(bytes).context("Error deserializing in-memory store")?; + e3_utils::deserialize_exact(bytes).context("Error deserializing in-memory store")?; let mut db = Hamt::new(); for (k, v) in map { db = db.insert(k, v); diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index 2892182f28..af76cccbb1 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -163,7 +163,7 @@ where if bytes == [0] { return Ok(None); } - Ok(Some(bincode::deserialize(&bytes)?)) + Ok(Some(e3_utils::deserialize_exact(&bytes)?)) } fn write_to_store(&self) { diff --git a/crates/events/src/interfold_event/mod.rs b/crates/events/src/interfold_event/mod.rs index e0a08943c0..3594777075 100644 --- a/crates/events/src/interfold_event/mod.rs +++ b/crates/events/src/interfold_event/mod.rs @@ -172,7 +172,9 @@ use std::{ hash::Hash, }; -// In crates/events/src/interfold_event/mod.rs +/// Commit-log records may be larger than the network's 10 MiB transport frame, but never exceed +/// the 32 MiB durable record cap configured by `e3-data`. +pub const MAX_ENCODED_EVENT_BYTES: u64 = 32 * 1024 * 1024; /// Macro to generate EventType enum and implement From traits macro_rules! impl_event_types { @@ -395,7 +397,7 @@ where } pub fn from_bytes(bytes: &[u8]) -> Result { - bincode::deserialize(bytes) + e3_utils::deserialize_bounded(bytes, MAX_ENCODED_EVENT_BYTES) } pub fn split(self) -> (InterfoldEventData, u128) { @@ -476,6 +478,26 @@ impl TryFrom> for InterfoldEvent { } } +#[cfg(test)] +mod serialization_tests { + use super::*; + + #[test] + fn event_decode_rejects_trailing_bytes() { + let event = InterfoldEvent::::new_with_timestamp( + TestEvent::new("bounded", 1).into(), + None, + 1, + None, + EventSource::Local, + ); + let mut bytes = event.to_bytes().unwrap(); + bytes.push(0); + + assert!(InterfoldEvent::::from_bytes(&bytes).is_err()); + } +} + #[cfg(feature = "test-helpers")] impl InterfoldEvent { /// test-helpers only utility function to create a new sequenced event diff --git a/crates/net/src/actors/net_sync_manager.rs b/crates/net/src/actors/net_sync_manager.rs index 678811ce6a..9f4a8ca60d 100644 --- a/crates/net/src/actors/net_sync_manager.rs +++ b/crates/net/src/actors/net_sync_manager.rs @@ -33,6 +33,7 @@ use crate::{ fetch_all_batched_events_with_budget, FetchEventsSince, SyncFetchBudget, }, sync_coordinator::sync_scan_limit, + wire::{decode, MAX_DIRECT_MESSAGE_BYTES}, EventTranslationService, NetReadiness, ReadinessDecision, SyncBatchOutcome, }, events::{ @@ -83,7 +84,7 @@ impl TryFrom> for SyncResponseValue { type Error = anyhow::Error; fn try_from(value: Vec) -> Result { - bincode::deserialize(&value).context("failed to deserialize sync response") + decode(&value, MAX_DIRECT_MESSAGE_BYTES).context("failed to deserialize sync response") } } diff --git a/crates/net/src/domain/event_conversion.rs b/crates/net/src/domain/event_conversion.rs index a2434a720f..450607a67b 100644 --- a/crates/net/src/domain/event_conversion.rs +++ b/crates/net/src/domain/event_conversion.rs @@ -13,6 +13,8 @@ use e3_utils::ArcBytes; use serde::{Deserialize, Serialize}; use tracing::{debug, info}; +use super::wire::{decode, MAX_DHT_DOCUMENT_BYTES}; + /// Wire representation of a document that is published to / received from the network. /// /// This is the serialized payload stored in the DHT. Disambiguation between the document @@ -30,7 +32,7 @@ impl ReceivableDocument { } pub fn from_bytes(bytes: &[u8]) -> Result { - bincode::deserialize(bytes) + decode(bytes, MAX_DHT_DOCUMENT_BYTES) } } diff --git a/crates/net/src/domain/mod.rs b/crates/net/src/domain/mod.rs index 65c5c9f72c..ad90222668 100644 --- a/crates/net/src/domain/mod.rs +++ b/crates/net/src/domain/mod.rs @@ -18,6 +18,7 @@ pub(crate) mod net_event_batch; mod network_status; pub(crate) mod peer_failure_tracker; pub(crate) mod sync_coordinator; +pub(crate) mod wire; pub use document_publishing::{datetime_to_instant_from_now, DocumentPublishingService}; pub use event_conversion::{EventConversionService, IncomingDocument}; diff --git a/crates/net/src/domain/net_event_batch.rs b/crates/net/src/domain/net_event_batch.rs index daa70a8d70..6dcba5107a 100644 --- a/crates/net/src/domain/net_event_batch.rs +++ b/crates/net/src/domain/net_event_batch.rs @@ -17,6 +17,7 @@ use tracing::info; use crate::{ direct_requester::{DirectRequester, WithPeer, WithoutPeer}, domain::sync_coordinator::effective_sync_limit, + domain::wire::{decode, MAX_DIRECT_MESSAGE_BYTES}, events::PeerTarget, }; @@ -131,7 +132,7 @@ where type Error = anyhow::Error; fn try_from(value: Vec) -> Result { - bincode::deserialize(&value).context("failed to deserialize EventBatch") + decode(&value, MAX_DIRECT_MESSAGE_BYTES).context("failed to deserialize EventBatch") } } @@ -187,7 +188,7 @@ impl TryFrom> for FetchEventsSince { type Error = anyhow::Error; fn try_from(value: Vec) -> Result { - bincode::deserialize(&value).context("failed to deserialize FetchEventsSince") + decode(&value, MAX_DIRECT_MESSAGE_BYTES).context("failed to deserialize FetchEventsSince") } } diff --git a/crates/net/src/domain/wire.rs b/crates/net/src/domain/wire.rs new file mode 100644 index 0000000000..5bd35a9047 --- /dev/null +++ b/crates/net/src/domain/wire.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use bincode::Error; +use serde::de::DeserializeOwned; + +pub(crate) const MAX_GOSSIP_BYTES: usize = 10 * 1024 * 1024; +pub(crate) const MAX_DIRECT_MESSAGE_BYTES: usize = 10 * 1024 * 1024; +pub(crate) const MAX_DHT_DOCUMENT_BYTES: usize = 25 * 1024 * 1024; + +pub(crate) fn decode(bytes: &[u8], max_bytes: usize) -> Result { + let max_bytes = + u64::try_from(max_bytes).map_err(|_| Box::new(bincode::ErrorKind::SizeLimit))?; + e3_utils::deserialize_bounded(bytes, max_bytes) +} diff --git a/crates/net/src/events.rs b/crates/net/src/events.rs index b59e2629e2..4562ac355f 100644 --- a/crates/net/src/events.rs +++ b/crates/net/src/events.rs @@ -4,7 +4,11 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{direct_responder::DirectResponder, ContentHash}; +use crate::{ + direct_responder::DirectResponder, + domain::wire::{decode, MAX_GOSSIP_BYTES}, + ContentHash, +}; use actix::Message; use anyhow::{anyhow, bail, Context, Result}; use e3_events::{ @@ -49,7 +53,7 @@ impl GossipData { } pub fn from_bytes(bytes: &[u8]) -> Result { - bincode::deserialize(bytes).context("Could not deserialize GossipData") + decode(bytes, MAX_GOSSIP_BYTES).context("Could not deserialize GossipData") } } @@ -362,7 +366,8 @@ impl DocumentPublishedNotification { } pub fn from_bytes(bytes: &[u8]) -> Result { - bincode::deserialize(bytes).context("Could not deserialize DocumentPublishedNotification") + decode(bytes, MAX_GOSSIP_BYTES) + .context("Could not deserialize DocumentPublishedNotification") } } @@ -502,4 +507,21 @@ mod tests { Ok(()) } + + #[test] + fn gossip_decode_rejects_forged_collection_length() { + let mut bytes = 0_u32.to_le_bytes().to_vec(); + bytes.extend_from_slice(&u64::MAX.to_le_bytes()); + + assert!(GossipData::from_bytes(&bytes).is_err()); + } + + #[test] + fn gossip_decode_rejects_trailing_bytes() { + let mut bytes = GossipData::GossipBytes(vec![1, 2, 3]).to_bytes().unwrap(); + bytes.push(0); + + let error = GossipData::from_bytes(&bytes).unwrap_err(); + assert!(format!("{error:#}").contains("trailing bytes")); + } } diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index 1f64fe0a0b..4be72a78b0 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -14,7 +14,11 @@ use crate::{ }; use crate::{ direct_responder::{ChannelType, DirectResponder}, - domain::{correlator::Correlator, peer_failure_tracker::PeerFailureTracker}, + domain::{ + correlator::Correlator, + peer_failure_tracker::PeerFailureTracker, + wire::{MAX_DHT_DOCUMENT_BYTES, MAX_GOSSIP_BYTES}, + }, events::{IncomingResponse, OutgoingRequest, ProtocolResponse}, keypair::Libp2pKeypair, net_interface_handle::NetInterfaceHandle, @@ -57,9 +61,7 @@ use tracing::{debug, error, info, trace, warn}; const PROTOCOL_NAME: StreamProtocol = StreamProtocol::new("/interfold/kad/1.0.0"); const MAX_KADEMLIA_PAYLOAD_MB: usize = 100; -const MAX_KADEMLIA_RECORD_MB: usize = 25; // Largest record: ~21MB ThresholdShare with prod params const DHT_MAX_RECORDS: usize = 4096; -const MAX_GOSSIP_MSG_SIZE_KB: usize = 10240; // 10MB — prod params C6 proofs are ~4.6MB const MAX_CONSECUTIVE_DIAL_FAILURES: u32 = 40; const EVENT_CHANNEL_SIZE: usize = 1000; const CMD_CHANNEL_SIZE: usize = 1000; @@ -294,7 +296,7 @@ fn create_behaviour( let gossipsub_config = gossipsub::ConfigBuilder::default() .heartbeat_interval(Duration::from_secs(10)) - .max_transmit_size(MAX_GOSSIP_MSG_SIZE_KB * 1024) + .max_transmit_size(MAX_GOSSIP_BYTES) .validation_mode(gossipsub::ValidationMode::Strict) .build() .map_err(Error::other)?; @@ -319,7 +321,7 @@ fn create_behaviour( .set_query_timeout(Duration::from_secs(30)); let store_config = MemoryStoreConfig { max_records: DHT_MAX_RECORDS, - max_value_bytes: MAX_KADEMLIA_RECORD_MB * 1024 * 1024, + max_value_bytes: MAX_DHT_DOCUMENT_BYTES, max_providers_per_key: usize::MAX, max_provided_keys: DHT_MAX_RECORDS, }; diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index cebe484273..97a6a10ba5 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -10,6 +10,7 @@ repository.workspace = true actix.workspace = true alloy.workspace = true anyhow.workspace = true +bincode.workspace = true derivative.workspace = true rand.workspace = true hex.workspace = true diff --git a/crates/utils/src/bounded_bincode.rs b/crates/utils/src/bounded_bincode.rs new file mode 100644 index 0000000000..1e9b83ab4a --- /dev/null +++ b/crates/utils/src/bounded_bincode.rs @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use bincode::{DefaultOptions, Error, ErrorKind, Options}; +use serde::de::DeserializeOwned; +use std::io::Cursor; + +/// Decode the repository's legacy bincode-1 fixed-integer format with an explicit byte budget. +/// +/// `bincode::deserialize` is unlimited and accepts trailing bytes. The slice-based `Options` +/// decoder also disables its configured limit internally, so this helper deliberately uses the +/// reader API and verifies complete consumption itself. +pub fn deserialize_bounded(bytes: &[u8], max_bytes: u64) -> Result { + let input_len = u64::try_from(bytes.len()).map_err(|_| Box::new(ErrorKind::SizeLimit))?; + if input_len > max_bytes { + return Err(Box::new(ErrorKind::SizeLimit)); + } + + let mut reader = Cursor::new(bytes); + let value = DefaultOptions::new() + .with_fixint_encoding() + .with_limit(max_bytes) + .deserialize_from(&mut reader)?; + if reader.position() != input_len { + return Err(Box::new(ErrorKind::Custom(format!( + "trailing bytes after bincode value: consumed {}, input {input_len}", + reader.position() + )))); + } + Ok(value) +} + +/// Decode one exact bincode value while using its encoded length as the deserialization budget. +pub fn deserialize_exact(bytes: &[u8]) -> Result { + let input_len = u64::try_from(bytes.len()).map_err(|_| Box::new(ErrorKind::SizeLimit))?; + deserialize_bounded(bytes, input_len) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decodes_legacy_fixed_integer_format() { + let value = vec![1_u64, 2, 3]; + let bytes = bincode::serialize(&value).unwrap(); + assert_eq!( + deserialize_bounded::>(&bytes, bytes.len() as u64).unwrap(), + value + ); + } + + #[test] + fn rejects_input_over_budget() { + let bytes = bincode::serialize(&vec![1_u8; 32]).unwrap(); + assert!(matches!( + deserialize_bounded::>(&bytes, 16), + Err(error) if matches!(*error, ErrorKind::SizeLimit) + )); + } + + #[test] + fn rejects_forged_collection_length() { + let bytes = u64::MAX.to_le_bytes(); + assert!(deserialize_bounded::>(&bytes, 1024).is_err()); + } + + #[test] + fn rejects_trailing_bytes() { + let mut bytes = bincode::serialize(&7_u64).unwrap(); + bytes.push(0); + let error = deserialize_bounded::(&bytes, bytes.len() as u64).unwrap_err(); + assert!(error.to_string().contains("trailing bytes")); + } +} diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index d76f7b6b5e..e7df9f5035 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -7,6 +7,7 @@ extern crate self as e3_utils; // need this for e3_utils_derive to reference this crate pub mod actix; pub mod alloy; +pub mod bounded_bincode; pub mod constants; pub mod error; pub mod formatters; @@ -17,6 +18,7 @@ pub mod serde_bytes; pub mod utility_types; pub use actix::NotifySync; pub use alloy::*; +pub use bounded_bincode::{deserialize_bounded, deserialize_exact}; pub use constants::*; pub use e3_utils_derive::BytesSerde; pub use error::*; From dafe5a7e0e347700807301d38b5c49f18605328c Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 16:34:12 +0500 Subject: [PATCH 27/62] fix(net): reject expired DHT documents --- crates/net/src/actors/document_publisher.rs | 42 ++++++++++++++++++- crates/net/src/domain/document_publishing.rs | 44 +++++++++++++++----- 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/crates/net/src/actors/document_publisher.rs b/crates/net/src/actors/document_publisher.rs index 0e47fdb690..7ac2235e61 100644 --- a/crates/net/src/actors/document_publisher.rs +++ b/crates/net/src/actors/document_publisher.rs @@ -15,7 +15,7 @@ use crate::{ ContentHash, }; use actix::prelude::*; -use anyhow::Result; +use anyhow::{Context, Result}; use e3_events::{ prelude::*, trap, trap_fut, BusHandle, CiphernodeSelected, CorrelationId, DecryptionKeyShared, DocumentReceived, E3RequestComplete, E3id, EType, EncryptionKeyCreated, EventSource, EventType, @@ -248,7 +248,10 @@ pub async fn handle_publish_document_requested( ) -> Result<()> { let value = event.value; let key = ContentHash::from_content(&value); - let expires = datetime_to_instant_from_now(event.meta.expires_at); + let expires = Some( + datetime_to_instant_from_now(event.meta.expires_at) + .context("refusing to publish an expired DHT document")?, + ); retry_with_backoff( || { @@ -673,6 +676,41 @@ mod tests { Ok(()) } + + #[actix::test] + async fn expired_document_is_rejected_without_a_dht_write() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("expired-document"); + let (net_cmd_tx, mut net_cmd_rx) = mpsc::channel(1); + let (_net_evt_tx, net_evt_rx) = broadcast::channel(1); + let event = PublishDocumentRequested { + meta: DocumentMeta::new( + E3id::new("expired", 1), + DocumentKind::TrBFV, + vec![], + Some(Utc::now() - chrono::Duration::seconds(1)), + ), + value: ArcBytes::from_bytes(b"stale"), + }; + + let error = handle_publish_document_requested( + net_cmd_tx, + Arc::new(net_evt_rx), + event, + "topic", + bus, + ) + .await + .unwrap_err(); + + assert!(format!("{error:#}").contains("expiry is not in the future")); + assert!(matches!( + net_cmd_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty | mpsc::error::TryRecvError::Disconnected) + )); + Ok(()) + } + #[actix::test] async fn test_get_document_fails_with_exponential_backoff() -> Result<()> { let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _, errors, _) = diff --git a/crates/net/src/domain/document_publishing.rs b/crates/net/src/domain/document_publishing.rs index 40993ef053..b315a5a03f 100644 --- a/crates/net/src/domain/document_publishing.rs +++ b/crates/net/src/domain/document_publishing.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use std::{collections::HashMap, time::Instant}; +use std::{collections::HashMap, fmt, time::Instant}; use chrono::{DateTime, Utc}; use e3_events::{E3id, PartyId}; @@ -84,20 +84,39 @@ impl DocumentPublishingService { } } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DocumentExpiryError { + AlreadyExpired, + OutOfRange, +} + +impl fmt::Display for DocumentExpiryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::AlreadyExpired => formatter.write_str("document expiry is not in the future"), + Self::OutOfRange => formatter.write_str("document expiry is outside Instant range"), + } + } +} + +impl std::error::Error for DocumentExpiryError {} + /// Convert a future UTC datetime into a monotonic [`Instant`] relative to now. -/// -/// Returns `None` if the target has already passed. -pub fn datetime_to_instant_from_now(target: DateTime) -> Option { +pub fn datetime_to_instant_from_now(target: DateTime) -> Result { let now_datetime = Utc::now(); let now_instant = Instant::now(); if target <= now_datetime { - return None; // Already expired + return Err(DocumentExpiryError::AlreadyExpired); } let duration = target.signed_duration_since(now_datetime); - let std_duration = duration.to_std().ok()?; - now_instant.checked_add(std_duration) + let std_duration = duration + .to_std() + .map_err(|_| DocumentExpiryError::OutOfRange)?; + now_instant + .checked_add(std_duration) + .ok_or(DocumentExpiryError::OutOfRange) } #[cfg(test)] @@ -157,12 +176,15 @@ mod tests { } #[test] - fn datetime_helper_is_none_for_past() { - assert!(datetime_to_instant_from_now(Utc::now() - chrono::Duration::days(1)).is_none()); + fn datetime_helper_rejects_past_expiry() { + assert_eq!( + datetime_to_instant_from_now(Utc::now() - chrono::Duration::days(1)), + Err(DocumentExpiryError::AlreadyExpired) + ); } #[test] - fn datetime_helper_is_some_for_future() { - assert!(datetime_to_instant_from_now(Utc::now() + chrono::Duration::days(1)).is_some()); + fn datetime_helper_accepts_future_expiry() { + assert!(datetime_to_instant_from_now(Utc::now() + chrono::Duration::days(1)).is_ok()); } } From 073c44c74a4de4408bb583ccadb52da76e36ca09 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 16:38:42 +0500 Subject: [PATCH 28/62] fix(bfv): bind proofs to supported parameters --- crates/bfv-client/src/client.rs | 83 +++++++++++++++++++++-- crates/fhe-params/src/builder.rs | 26 +++++++ crates/fhe-params/src/lib.rs | 2 +- crates/fhe-params/src/presets.rs | 47 +++++++++++++ crates/fhe-params/src/search/constants.rs | 18 ++--- crates/fhe-params/src/search/prime.rs | 2 +- 6 files changed, 160 insertions(+), 18 deletions(-) diff --git a/crates/bfv-client/src/client.rs b/crates/bfv-client/src/client.rs index b3f709cd39..91cc11e5a5 100644 --- a/crates/bfv-client/src/client.rs +++ b/crates/bfv-client/src/client.rs @@ -5,7 +5,7 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::{anyhow, Result}; -use e3_fhe_params::{build_bfv_params_arc, DEFAULT_BFV_PRESET}; +use e3_fhe_params::{try_build_bfv_params_arc, BfvParamSet, BfvPreset}; use e3_zk_helpers::circuits::threshold::user_data_encryption::circuit::UserDataEncryptionCircuitData; use e3_zk_helpers::circuits::threshold::user_data_encryption::Inputs as UserDataEncryptionInputs; use e3_zk_helpers::circuits::Computation; @@ -14,6 +14,16 @@ use fhe::Error as FheError; use fhe_traits::{DeserializeParametrized, FheEncoder, FheEncrypter, Serialize}; use rand::rng; +fn build_client_params( + degree: usize, + plaintext_modulus: u64, + moduli: &[u64], + error1_variance: Option<&str>, +) -> Result> { + try_build_bfv_params_arc(degree, plaintext_modulus, moduli, error1_variance) + .map_err(|error| anyhow!("Invalid BFV parameters: {error}")) +} + /// Encrypt some data using BFV homomorphic encryption /// /// # Arguments @@ -42,7 +52,7 @@ pub fn bfv_encrypt( where Plaintext: for<'a> FheEncoder<&'a T, Error = FheError>, { - let params = build_bfv_params_arc(degree, plaintext_modulus, moduli, None); + let params = build_client_params(degree, plaintext_modulus, moduli, None)?; let pk = PublicKey::from_bytes(&public_key, ¶ms) .map_err(|e| anyhow!("Error deserializing public key:{e}"))?; @@ -93,7 +103,21 @@ pub fn bfv_verifiable_encrypt( where Plaintext: for<'a> FheEncoder<&'a T, Error = FheError>, { - let params = build_bfv_params_arc(degree, plaintext_modulus, &moduli, None); + let preset = BfvPreset::from_threshold_parameters(degree, plaintext_modulus, &moduli) + .ok_or_else(|| { + anyhow!( + "Unsupported BFV threshold parameters for verifiable encryption: degree={degree}, \ + plaintext_modulus={plaintext_modulus}, moduli_count={}", + moduli.len() + ) + })?; + let preset_parameters = BfvParamSet::from(preset); + let params = build_client_params( + degree, + plaintext_modulus, + &moduli, + preset_parameters.error1_variance, + )?; let pk = PublicKey::from_bytes(&public_key, ¶ms) .map_err(|e| anyhow!("Error deserializing public key: {}", e))?; @@ -102,7 +126,7 @@ where .map_err(|e: FheError| anyhow!("Error encoding plaintext: {}", e))?; let inputs = UserDataEncryptionInputs::compute( - DEFAULT_BFV_PRESET, + preset, &UserDataEncryptionCircuitData { public_key: pk, plaintext, @@ -132,7 +156,7 @@ pub fn generate_public_key( plaintext_modulus: u64, moduli: Vec, ) -> Result> { - let params = build_bfv_params_arc(degree, plaintext_modulus, &moduli, None); + let params = build_client_params(degree, plaintext_modulus, &moduli, None)?; // Generate keys. let mut rng = rng(); @@ -150,7 +174,7 @@ pub fn compute_pk_commitment( ) -> Result<[u8; 32]> { use e3_zk_helpers::circuits::threshold::user_data_encryption::utils::compute_public_key_commitment; - let params = build_bfv_params_arc(degree, plaintext_modulus, &moduli, None); + let params = build_client_params(degree, plaintext_modulus, &moduli, None)?; let public_key = PublicKey::from_bytes(&public_key, ¶ms) .map_err(|e| anyhow!("Error deserializing public key: {}", e))?; @@ -169,7 +193,7 @@ pub fn compute_ct_commitment( ) -> Result<[u8; 32]> { use e3_zk_helpers::circuits::threshold::user_data_encryption::utils::compute_ciphertext_commitment; - let params = build_bfv_params_arc(degree, plaintext_modulus, &moduli, None); + let params = build_client_params(degree, plaintext_modulus, &moduli, None)?; let ct = Ciphertext::from_bytes(&ct, ¶ms) .map_err(|e| anyhow!("Error deserializing ciphertext: {}", e))?; @@ -188,6 +212,51 @@ mod tests { use super::*; + #[test] + fn verifiable_encryption_rejects_noncanonical_parameter_tuple_before_key_decode() { + let parameters: BfvParamSet = DEFAULT_BFV_PRESET.into(); + let mut moduli = parameters.moduli.to_vec(); + moduli[0] ^= 1; + + let error = bfv_verifiable_encrypt( + [1_u64], + Vec::new(), + parameters.degree, + parameters.plaintext_modulus, + moduli, + ) + .unwrap_err(); + + assert!(error + .to_string() + .contains("Unsupported BFV threshold parameters")); + } + + #[test] + fn public_key_generation_returns_error_for_invalid_parameters() { + let error = generate_public_key(7, 17, vec![97]).unwrap_err(); + assert!(error.to_string().contains("Invalid BFV parameters")); + } + + #[test] + fn verifiable_parameter_builder_uses_the_canonical_error_variance() { + for preset in BfvPreset::PAIR_PRESETS { + let parameters = BfvParamSet::from(preset); + let built = build_client_params( + parameters.degree, + parameters.plaintext_modulus, + parameters.moduli, + parameters.error1_variance, + ) + .unwrap(); + + assert_eq!( + built.get_error1_variance().to_string(), + parameters.error1_variance.unwrap() + ); + } + } + #[test] fn test_bfv_encrypt_a64() { use fhe::bfv::{Ciphertext, PublicKey, SecretKey}; diff --git a/crates/fhe-params/src/builder.rs b/crates/fhe-params/src/builder.rs index 4936d1f591..4dcd4792be 100644 --- a/crates/fhe-params/src/builder.rs +++ b/crates/fhe-params/src/builder.rs @@ -122,6 +122,26 @@ pub fn build_bfv_params_arc( .unwrap_or_else(|e| panic!("Failed to build BFV Parameters wrapped in Arc: {}", e)) } +/// Fallible BFV parameter construction for public or otherwise untrusted inputs. +pub fn try_build_bfv_params_arc( + degree: usize, + plaintext_modulus: u64, + moduli: &[u64], + error1_variance: Option<&str>, +) -> Result, fhe::Error> { + let mut builder = BfvParametersBuilder::new(); + builder + .set_degree(degree) + .set_plaintext_modulus(plaintext_modulus) + .set_moduli(moduli); + + if let Some(error1_variance) = error1_variance { + builder.set_error1_variance_str(error1_variance)?; + } + + builder.build_arc() +} + #[cfg(test)] mod tests { use super::*; @@ -167,6 +187,12 @@ mod tests { ); } + #[test] + fn fallible_builder_rejects_invalid_degree_without_panicking() { + let result = try_build_bfv_params_arc(7, 17, &[97], None); + assert!(result.is_err()); + } + #[test] fn test_build_secure_threshold_params() { // Test building threshold params using secure threshold preset constants diff --git a/crates/fhe-params/src/lib.rs b/crates/fhe-params/src/lib.rs index bd6bdf2d76..889eb1f4e5 100644 --- a/crates/fhe-params/src/lib.rs +++ b/crates/fhe-params/src/lib.rs @@ -16,7 +16,7 @@ pub mod search; pub use builder::{ build_bfv_params, build_bfv_params_arc, build_bfv_params_from_set, - build_bfv_params_from_set_arc, build_pair_for_preset, + build_bfv_params_from_set_arc, build_pair_for_preset, try_build_bfv_params_arc, }; pub use crp::{create_deterministic_crp_from_default_seed, create_deterministic_crp_from_seed}; #[cfg(feature = "abi-encoding")] diff --git a/crates/fhe-params/src/presets.rs b/crates/fhe-params/src/presets.rs index 105dbd1361..023d188d77 100644 --- a/crates/fhe-params/src/presets.rs +++ b/crates/fhe-params/src/presets.rs @@ -345,6 +345,20 @@ impl BfvPreset { Self::PAIR_PRESETS.contains(self) } + /// Resolve an exact threshold parameter tuple to the preset whose circuit statement uses it. + pub fn from_threshold_parameters( + degree: usize, + plaintext_modulus: u64, + moduli: &[u64], + ) -> Option { + Self::PAIR_PRESETS.iter().copied().find(|preset| { + let parameters = BfvParamSet::from(*preset); + parameters.degree == degree + && parameters.plaintext_modulus == plaintext_modulus + && parameters.moduli == moduli + }) + } + /// Returns the DKG preset that pairs with this threshold preset. /// /// Used when you have a threshold preset (e.g. for encryption/decryption) and need @@ -525,6 +539,39 @@ mod tests { } } + #[test] + fn threshold_parameter_tuple_resolves_only_exact_supported_presets() { + for preset in BfvPreset::PAIR_PRESETS { + let parameters = BfvParamSet::from(preset); + assert_eq!( + BfvPreset::from_threshold_parameters( + parameters.degree, + parameters.plaintext_modulus, + parameters.moduli, + ), + Some(preset) + ); + } + + let dkg = BfvParamSet::from(BfvPreset::InsecureDkg512); + assert_eq!( + BfvPreset::from_threshold_parameters(dkg.degree, dkg.plaintext_modulus, dkg.moduli,), + None + ); + + let threshold = BfvParamSet::from(BfvPreset::InsecureThreshold512); + let mut changed_moduli = threshold.moduli.to_vec(); + changed_moduli[0] ^= 1; + assert_eq!( + BfvPreset::from_threshold_parameters( + threshold.degree, + threshold.plaintext_modulus, + &changed_moduli, + ), + None + ); + } + #[test] fn build_pair_matches_expected_params() { let (threshold, dkg) = BfvPreset::InsecureThreshold512.build_pair().unwrap(); diff --git a/crates/fhe-params/src/search/constants.rs b/crates/fhe-params/src/search/constants.rs index 77f2817b2c..9ad62a44fb 100644 --- a/crates/fhe-params/src/search/constants.rs +++ b/crates/fhe-params/src/search/constants.rs @@ -464,6 +464,13 @@ pub const NTT_PRIMES_BY_BITS: &[(u8, &[&str])] = &[ ), ]; +/// Starting polynomial degree (power of 2) for parameter search +pub const D_POW2_START: u64 = 256; +/// Maximum polynomial degree (power of 2) to search up to +pub const D_POW2_MAX: u64 = 32768; +/// Maximum number of votes/plaintext additions (k_plain_eff) +pub const K_MAX: u128 = 1u128 << 25; // 33,554,432 + #[cfg(test)] mod tests { use super::NTT_PRIMES_BY_BITS; @@ -479,13 +486,13 @@ mod tests { return false; } for p in [2u64, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37] { - if n % p == 0 { + if n.is_multiple_of(p) { return n == p; } } let mut d = n - 1; let mut r = 0; - while d % 2 == 0 { + while d.is_multiple_of(2) { d /= 2; r += 1; } @@ -544,10 +551,3 @@ mod tests { } } } - -/// Starting polynomial degree (power of 2) for parameter search -pub const D_POW2_START: u64 = 256; -/// Maximum polynomial degree (power of 2) to search up to -pub const D_POW2_MAX: u64 = 32768; -/// Maximum number of votes/plaintext additions (k_plain_eff) -pub const K_MAX: u128 = 1u128 << 25; // 33,554,432 diff --git a/crates/fhe-params/src/search/prime.rs b/crates/fhe-params/src/search/prime.rs index 98ccaa90cf..9d117fc7bd 100644 --- a/crates/fhe-params/src/search/prime.rs +++ b/crates/fhe-params/src/search/prime.rs @@ -35,7 +35,7 @@ fn build_prime_items_with_filter(filter: BitFilter) -> Vec { if filter(*bits) { continue; } - for &phex in arr.into_iter() { + for &phex in arr.iter() { let v = parse_hex_big(phex); vec.push(PrimeItem { bitlen: *bits, From a874c7846e8bc3b193fc591510fd63096db6a1bb Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 16:43:09 +0500 Subject: [PATCH 29/62] fix(evm): fail closed on missing block timestamps --- crates/evm/src/actors/evm_chain_gateway.rs | 28 +++- crates/evm/src/actors/evm_parser.rs | 148 +++++++++++++++--- crates/evm/src/actors/evm_read_interface.rs | 13 +- crates/evm/src/actors/evm_router.rs | 13 +- crates/evm/src/actors/log_fetcher.rs | 126 ++++++++++++--- .../evm/src/domain/historical_order_fixer.rs | 12 ++ crates/evm/src/messages.rs | 23 +++ 7 files changed, 310 insertions(+), 53 deletions(-) diff --git a/crates/evm/src/actors/evm_chain_gateway.rs b/crates/evm/src/actors/evm_chain_gateway.rs index f6cf5139f5..5ce870b06d 100644 --- a/crates/evm/src/actors/evm_chain_gateway.rs +++ b/crates/evm/src/actors/evm_chain_gateway.rs @@ -161,6 +161,12 @@ impl EvmChainGateway { InterfoldEvmEvent::Log(_) => { bail!("EvmChainGateway received an unparsed EVM log") } + InterfoldEvmEvent::Rejected(rejected) => bail!( + "chain {} rejected provider log {}: {}", + rejected.chain_id, + rejected.id, + rejected.reason + ), InterfoldEvmEvent::Processed(_) => { bail!("EvmChainGateway received an internal ordering marker") } @@ -242,7 +248,7 @@ impl Handler for EvmChainGateway { #[cfg(test)] mod tests { - use crate::EvmEvent; + use crate::{EvmEvent, EvmLogRejected}; use super::*; use e3_ciphernode_builder::EventSystem; @@ -255,6 +261,26 @@ mod tests { tx: mpsc::UnboundedSender, } + #[actix::test] + async fn rejected_log_fails_gateway_readiness() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-rejected-log"); + let gateway = EvmChainGateway::setup_with_readiness(&bus); + + gateway + .addr() + .send(InterfoldEvmEvent::Rejected(EvmLogRejected::new( + CorrelationId::new(), + 1, + "malformed historical log", + ))) + .await?; + + let error = gateway.wait_until_live().await.unwrap_err(); + assert!(error.to_string().contains("malformed historical log")); + Ok(()) + } + impl Actor for SyncEventCollector { type Context = actix::Context; } diff --git a/crates/evm/src/actors/evm_parser.rs b/crates/evm/src/actors/evm_parser.rs index 31df0cf44c..c8f0f43cb2 100644 --- a/crates/evm/src/actors/evm_parser.rs +++ b/crates/evm/src/actors/evm_parser.rs @@ -6,12 +6,13 @@ use actix::{Actor, Handler}; use alloy::primitives::{LogData, B256}; +use anyhow::{Context as _, Result}; use e3_events::InterfoldEventData; use e3_utils::MAILBOX_LIMIT; -use tracing::debug; +use tracing::{debug, error}; use crate::domain::log_timestamp::from_log_chain_id_to_ts; -use crate::messages::{EvmEvent, EvmEventProcessor, EvmLog, InterfoldEvmEvent}; +use crate::messages::{EvmEvent, EvmEventProcessor, EvmLog, EvmLogRejected, InterfoldEvmEvent}; pub type ExtractorFn = fn(&LogData, &[B256], u64) -> Option; @@ -36,30 +37,44 @@ impl EvmParser { } } +fn parse_log(log: EvmLog, extractor: ExtractorFn) -> Result { + let block = log.log.block_number.context( + "provider log is missing its block number; pending or malformed logs cannot be ordered", + )?; + let log_index = log.log.log_index.context( + "provider log is missing its log index; malformed logs cannot be ordered deterministically", + )?; + let event = extractor(log.log.data(), log.log.topics(), log.chain_id).context( + "contract log matched a configured address but could not be decoded; refusing to advance", + )?; + let timestamp = from_log_chain_id_to_ts(log.timestamp, log_index, log.chain_id); + Ok(EvmEvent::new(log.id, event, block, timestamp, log.chain_id)) +} + impl Handler for EvmParser { type Result = (); fn handle(&mut self, msg: InterfoldEvmEvent, _ctx: &mut Self::Context) -> Self::Result { match msg.clone() { - InterfoldEvmEvent::Log(EvmLog { - log, - chain_id, - id, - timestamp, - }) => { + InterfoldEvmEvent::Log(log) => { debug!("processing event({})", msg.get_id()); - let extractor = self.extractor; - - if let Some(event) = extractor(log.data(), log.topics(), chain_id) { - let err = "Log should always have metadata because we listen to non-pending blocks. If you are seeing this it is likely because there is an issue with how we are subscribing to blocks"; - let block = log.block_number.expect(err); - let log_index = log.log_index.expect(err); - let ts = from_log_chain_id_to_ts(timestamp, log_index, chain_id); - self.next.do_send(InterfoldEvmEvent::Event(EvmEvent::new( - // note we use the id from the log event above! - id, event, block, ts, chain_id, - ))) - } else { - self.next.do_send(InterfoldEvmEvent::Processed(id)) + let id = log.id; + let chain_id = log.chain_id; + match parse_log(log, self.extractor) { + Ok(event) => self.next.do_send(InterfoldEvmEvent::Event(event)), + Err(parse_error) => { + error!( + %id, + chain_id, + error = %parse_error, + "Rejecting EVM log and failing the chain ingestion pipeline" + ); + self.next + .do_send(InterfoldEvmEvent::Rejected(EvmLogRejected::new( + id, + chain_id, + parse_error.to_string(), + ))); + } } } hist @ InterfoldEvmEvent::HistoricalSyncComplete(..) => self.next.do_send(hist), @@ -67,3 +82,94 @@ impl Handler for EvmParser { } } } + +#[cfg(test)] +mod tests { + use super::*; + use actix::{Actor, Context, Handler}; + use alloy::rpc::types::Log; + use e3_events::TestEvent; + use tokio::sync::mpsc; + + fn test_extractor(_: &LogData, _: &[B256], _: u64) -> Option { + Some(TestEvent::new("parsed", 1).into()) + } + + fn rejected_extractor(_: &LogData, _: &[B256], _: u64) -> Option { + None + } + + fn log(block_number: Option, log_index: Option) -> EvmLog { + EvmLog::new( + Log { + block_number, + log_index, + ..Default::default() + }, + 1, + 10, + ) + } + + #[test] + fn parser_requires_block_number() { + let error = parse_log(log(None, Some(0)), test_extractor).unwrap_err(); + assert!(error.to_string().contains("missing its block number")); + } + + #[test] + fn parser_requires_log_index() { + let error = parse_log(log(Some(1), None), test_extractor).unwrap_err(); + assert!(error.to_string().contains("missing its log index")); + } + + #[test] + fn parser_rejects_failed_contract_decode() { + let error = parse_log(log(Some(1), Some(0)), rejected_extractor).unwrap_err(); + assert!(error.to_string().contains("could not be decoded")); + } + + #[test] + fn parser_converts_valid_log() { + let source = log(Some(7), Some(3)); + let id = source.id; + let event = parse_log(source, test_extractor).unwrap(); + assert_eq!(event.get_id(), id); + let (_, _, block) = event.split(); + assert_eq!(block, 7); + } + + struct Collector(mpsc::UnboundedSender); + + impl Actor for Collector { + type Context = Context; + } + + impl Handler for Collector { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvmEvent, _: &mut Self::Context) { + let _ = self.0.send(msg); + } + } + + #[actix::test] + async fn parser_propagates_an_explicit_rejection() { + let (tx, mut rx) = mpsc::unbounded_channel(); + let collector = Collector(tx).start().recipient(); + let parser = EvmParser::new(&collector, rejected_extractor).start(); + let malformed = log(Some(7), Some(3)); + let expected_id = malformed.id; + + parser + .send(InterfoldEvmEvent::Log(malformed)) + .await + .unwrap(); + + let rejected = rx.recv().await.expect("parser rejection"); + assert!(matches!( + rejected, + InterfoldEvmEvent::Rejected(EvmLogRejected { id, chain_id: 1, .. }) if id == expected_id + )); + } +} diff --git a/crates/evm/src/actors/evm_read_interface.rs b/crates/evm/src/actors/evm_read_interface.rs index 04e15e5a8a..95e03ceb93 100644 --- a/crates/evm/src/actors/evm_read_interface.rs +++ b/crates/evm/src/actors/evm_read_interface.rs @@ -414,11 +414,20 @@ async fn stream_from_evm( maybe_log = stream.next() => { match maybe_log { Some(log) => { - let _ = process_live_log( + if let Err(error) = process_live_log( current_provider.provider(), log, chain_id, &next, &mut timestamp_tracker, &mut last_block, filters.confirmations(), - ).await; + ).await { + consecutive_failures += 1; + warn!( + chain_id, + error = %error, + consecutive_failures, + "Live log rejected; reconnecting for canonical backfill" + ); + break; + } } None => { // Stream ended (server-side close, idle timeout, etc.) diff --git a/crates/evm/src/actors/evm_router.rs b/crates/evm/src/actors/evm_router.rs index e8454af418..393d3559c1 100644 --- a/crates/evm/src/actors/evm_router.rs +++ b/crates/evm/src/actors/evm_router.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::messages::{EvmEventProcessor, EvmLog, InterfoldEvmEvent}; +use crate::messages::{EvmEventProcessor, EvmLog, EvmLogRejected, InterfoldEvmEvent}; use actix::{Actor, Handler}; use alloy_primitives::Address; use e3_utils::MAILBOX_LIMIT; @@ -59,7 +59,7 @@ impl Handler for EvmRouter { fn handle(&mut self, msg: InterfoldEvmEvent, _: &mut Self::Context) -> Self::Result { match msg.clone() { // Take all log events and route them - InterfoldEvmEvent::Log(EvmLog { log, .. }) => { + InterfoldEvmEvent::Log(EvmLog { log, chain_id, .. }) => { let address = log.address(); if let Some(dest) = self.routing_table.get(&address) { debug!("Found address {address} in routing table forwarding to destination."); @@ -68,7 +68,14 @@ impl Handler for EvmRouter { error!( "Could not find a route for log with address = {:?}", log.address() - ) + ); + if let Some(fallback) = self.fallback.clone() { + fallback.do_send(InterfoldEvmEvent::Rejected(EvmLogRejected::new( + msg.get_id(), + chain_id, + format!("no configured route for provider log address {address}"), + ))); + } } } _ => { diff --git a/crates/evm/src/actors/log_fetcher.rs b/crates/evm/src/actors/log_fetcher.rs index 993b7f8864..192eb031e4 100644 --- a/crates/evm/src/actors/log_fetcher.rs +++ b/crates/evm/src/actors/log_fetcher.rs @@ -7,10 +7,10 @@ use crate::messages::{EvmEventProcessor, EvmLog, InterfoldEvmEvent}; use alloy::providers::Provider; use alloy::rpc::types::{Filter, Log}; -use anyhow::anyhow; +use anyhow::{anyhow, Context as _}; use async_trait::async_trait; use e3_events::CorrelationId; -use tracing::{debug, error, info, warn}; +use tracing::{debug, info, warn}; const GET_LOGS_CHUNK_SIZE: u64 = 10_000; const GET_LOGS_MAX_RETRIES: u32 = 3; @@ -21,7 +21,7 @@ const GET_LOGS_MAX_RETRIES: u32 = 3; pub(crate) trait LogProvider: Send + Sync { async fn fetch_logs(&self, filter: &Filter) -> Result, anyhow::Error>; async fn fetch_block_number(&self) -> Result; - async fn fetch_block_timestamp(&self, block_number: u64) -> Option; + async fn fetch_block_timestamp(&self, block_number: u64) -> Result; } #[async_trait] @@ -32,12 +32,12 @@ impl LogProvider for P { async fn fetch_block_number(&self) -> Result { self.get_block_number().await.map_err(|e| anyhow!("{}", e)) } - async fn fetch_block_timestamp(&self, block_number: u64) -> Option { + async fn fetch_block_timestamp(&self, block_number: u64) -> Result { self.get_block_by_number(block_number.into()) .await - .ok() - .flatten() - .map(|b| b.header.timestamp) + .map_err(|error| anyhow!("failed to fetch block {block_number}: {error}"))? + .map(|block| block.header.timestamp) + .ok_or_else(|| anyhow!("provider returned no block for height {block_number}")) } } @@ -47,13 +47,13 @@ pub(crate) async fn process_log( chain_id: u64, next: &EvmEventProcessor, timestamp_tracker: &mut TimestampTracker, -) -> CorrelationId { - let timestamp = timestamp_tracker.get(provider, log.block_number).await; +) -> Result { + let timestamp = timestamp_tracker.get(provider, log.block_number).await?; let evt = InterfoldEvmEvent::Log(EvmLog::new(log, chain_id, timestamp)); let id = evt.get_id(); debug!("Sending event({})", id); next.do_send(evt); - id + Ok(id) } /// Handle a log delivered by the subscription stream. @@ -70,7 +70,7 @@ pub(crate) async fn process_live_log( timestamp_tracker: &mut TimestampTracker, last_block: &mut u64, confirmations: u64, -) -> Option { +) -> Result, anyhow::Error> { if confirmations > 0 { debug!( chain_id, @@ -78,13 +78,15 @@ pub(crate) async fn process_live_log( confirmations, "Deferring live log to confirmed canonical backfill" ); - return None; + return Ok(None); } - if let Some(block_number) = log.block_number { + let block_number = log.block_number; + let id = process_log(provider, log, chain_id, next, timestamp_tracker).await?; + if let Some(block_number) = block_number { *last_block = (*last_block).max(block_number); } - Some(process_log(provider, log, chain_id, next, timestamp_tracker).await) + Ok(Some(id)) } /// Fetch logs in chunks from `from_block` to `to_block` with retry logic per chunk. @@ -135,7 +137,7 @@ pub(crate) async fn fetch_logs_chunked( ); for log in logs { last_id = Some( - process_log(provider, log, chain_id, next, timestamp_tracker).await, + process_log(provider, log, chain_id, next, timestamp_tracker).await?, ); } success = true; @@ -237,22 +239,23 @@ impl TimestampTracker { Self { current: None } } - pub async fn get(&mut self, provider: &L, block_number: Option) -> u64 { - let Some(bn) = block_number else { - error!("BLOCK NUMBER NOT FOUND ON LOG!"); - return 0; - }; + pub async fn get( + &mut self, + provider: &L, + block_number: Option, + ) -> Result { + let bn = block_number.context("provider log is missing its block number")?; if let Some((cached_bn, ts)) = self.current { if bn == cached_bn { - return ts; + return Ok(ts); } } - let ts = provider.fetch_block_timestamp(bn).await.unwrap_or(0); + let ts = provider.fetch_block_timestamp(bn).await?; self.current = Some((bn, ts)); - ts + Ok(ts) } } @@ -272,6 +275,8 @@ mod tests { struct MockState { block_number: u64, log_responses: VecDeque, String>>, + timestamp_responses: VecDeque>, + timestamp_calls: u32, get_logs_calls: u32, } @@ -281,6 +286,8 @@ mod tests { inner: Arc::new(Mutex::new(MockState { block_number, log_responses: VecDeque::new(), + timestamp_responses: VecDeque::new(), + timestamp_calls: 0, get_logs_calls: 0, })), } @@ -302,6 +309,26 @@ mod tests { self.inner.lock().unwrap().get_logs_calls } + fn push_timestamp(&self, timestamp: u64) { + self.inner + .lock() + .unwrap() + .timestamp_responses + .push_back(Ok(timestamp)); + } + + fn push_timestamp_error(&self, message: &str) { + self.inner + .lock() + .unwrap() + .timestamp_responses + .push_back(Err(message.to_string())); + } + + fn timestamp_call_count(&self) -> u32 { + self.inner.lock().unwrap().timestamp_calls + } + fn set_block_number(&self, block_number: u64) { self.inner.lock().unwrap().block_number = block_number; } @@ -323,8 +350,14 @@ mod tests { Ok(self.inner.lock().unwrap().block_number) } - async fn fetch_block_timestamp(&self, _block_number: u64) -> Option { - Some(0) + async fn fetch_block_timestamp(&self, _block_number: u64) -> Result { + let mut state = self.inner.lock().unwrap(); + state.timestamp_calls += 1; + match state.timestamp_responses.pop_front() { + Some(Ok(timestamp)) => Ok(timestamp), + Some(Err(message)) => Err(anyhow!(message)), + None => Ok(0), + } } } @@ -373,6 +406,47 @@ mod tests { assert_eq!(mock.get_logs_call_count(), 0); } + #[actix::test] + async fn timestamp_failure_is_not_cached_as_zero() { + let mock = MockLogProvider::new(100); + mock.push_timestamp_error("RPC unavailable"); + mock.push_timestamp(1234); + let mut tracker = TimestampTracker::new(); + + let error = tracker.get(&mock, Some(100)).await.unwrap_err(); + assert!(error.to_string().contains("RPC unavailable")); + assert_eq!(tracker.get(&mock, Some(100)).await.unwrap(), 1234); + assert_eq!(tracker.get(&mock, Some(100)).await.unwrap(), 1234); + assert_eq!(mock.timestamp_call_count(), 2); + } + + #[actix::test] + async fn missing_log_block_number_is_rejected_without_rpc_fallback() { + let mock = MockLogProvider::new(100); + let mut tracker = TimestampTracker::new(); + + let error = tracker.get(&mock, None).await.unwrap_err(); + assert!(error.to_string().contains("missing its block number")); + assert_eq!(mock.timestamp_call_count(), 0); + } + + #[actix::test] + async fn timestamp_rpc_failure_prevents_log_dispatch() { + let mock = MockLogProvider::new(100); + mock.push_logs(vec![make_test_log(100)]); + mock.push_timestamp_error("timestamp RPC unavailable"); + let (next, mut receiver) = setup_collector(); + let mut tracker = TimestampTracker::new(); + + let error = fetch_logs_chunked(&mock, &Filter::new(), 100, 100, 1, &next, &mut tracker) + .await + .unwrap_err(); + + assert!(error.to_string().contains("timestamp RPC unavailable")); + tokio::task::yield_now().await; + assert!(receiver.try_recv().is_err()); + } + #[actix::test] async fn test_fetch_logs_single_chunk() { let mock = MockLogProvider::new(5000); @@ -569,7 +643,7 @@ mod tests { &mut last_block, 12, ) - .await; + .await?; assert!(delivered.is_none()); assert_eq!(last_block, 188); tokio::task::yield_now().await; diff --git a/crates/evm/src/domain/historical_order_fixer.rs b/crates/evm/src/domain/historical_order_fixer.rs index def7a3fd28..9a57fd1c14 100644 --- a/crates/evm/src/domain/historical_order_fixer.rs +++ b/crates/evm/src/domain/historical_order_fixer.rs @@ -137,4 +137,16 @@ mod tests { // A Processed marker for the referenced id releases the sync but is not forwarded assert_eq!(fixer.process(InterfoldEvmEvent::Processed(id)), vec![sync]); } + + #[test] + fn rejected_logs_are_forwarded_to_fail_the_gateway() { + let mut fixer = HistoricalOrderFixer::new(); + let rejected = InterfoldEvmEvent::Rejected(crate::messages::EvmLogRejected::new( + CorrelationId::new(), + 1, + "malformed", + )); + + assert_eq!(fixer.process(rejected.clone()), vec![rejected]); + } } diff --git a/crates/evm/src/messages.rs b/crates/evm/src/messages.rs index defbb3714a..553c3e8591 100644 --- a/crates/evm/src/messages.rs +++ b/crates/evm/src/messages.rs @@ -22,6 +22,26 @@ pub struct HistoricalSyncComplete { pub id: CorrelationId, } +/// Explicit negative acknowledgement for a provider log that cannot enter the +/// canonical event pipeline. Historical synchronization must fail closed on +/// this message rather than letting a later log advance its completion marker. +#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub struct EvmLogRejected { + pub id: CorrelationId, + pub chain_id: u64, + pub reason: String, +} + +impl EvmLogRejected { + pub fn new(id: CorrelationId, chain_id: u64, reason: impl Into) -> Self { + Self { + id, + chain_id, + reason: reason.into(), + } + } +} + impl HistoricalSyncComplete { pub fn new(chain_id: u64, prev_event: Option) -> Self { let id = CorrelationId::new(); @@ -97,6 +117,8 @@ pub enum InterfoldEvmEvent { Event(EvmEvent), /// Raw log data from the provider Log(EvmLog), + /// A raw log was rejected before it could become a canonical event. + Rejected(EvmLogRejected), /// Dummy event to report that an event was processed. This is required to ensure that the /// appropriate events are ordered correctly Processed(CorrelationId), @@ -107,6 +129,7 @@ impl InterfoldEvmEvent { match self { InterfoldEvmEvent::HistoricalSyncComplete(e) => e.get_id(), InterfoldEvmEvent::Log(e) => e.get_id(), + InterfoldEvmEvent::Rejected(e) => e.id, InterfoldEvmEvent::Event(e) => e.get_id(), InterfoldEvmEvent::Processed(id) => id.to_owned(), } From ff71d0e47c4e00324c0a2d7d2539b2417d9e6460 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 16:50:21 +0500 Subject: [PATCH 30/62] fix(evm): reject reverted transaction receipts --- crates/cli/src/ciphernode/license.rs | 4 ++ crates/cli/src/ciphernode/lifecycle.rs | 5 ++ crates/cli/src/ciphernode/tickets.rs | 3 + crates/cli/src/ciphernode/utils.rs | 22 +++--- crates/cli/src/faucet.rs | 1 + crates/evm-helpers/src/contracts.rs | 4 ++ .../evm/src/actors/ciphernode_registry_sol.rs | 5 +- crates/evm/src/actors/interfold_sol_writer.rs | 5 +- .../src/actors/slashing_manager_sol_writer.rs | 3 +- crates/utils/src/lib.rs | 2 + crates/utils/src/transaction.rs | 69 +++++++++++++++++++ 11 files changed, 109 insertions(+), 14 deletions(-) create mode 100644 crates/utils/src/transaction.rs diff --git a/crates/cli/src/ciphernode/license.rs b/crates/cli/src/ciphernode/license.rs index 8ad1b70474..47e7b5e30d 100644 --- a/crates/cli/src/ciphernode/license.rs +++ b/crates/cli/src/ciphernode/license.rs @@ -7,6 +7,7 @@ use alloy::primitives::U256; use anyhow::Result; use e3_console::{log, Console}; +use e3_utils::require_successful_receipt; use super::context::ChainContext; use super::utils::{ensure_allowance, parse_amount}; @@ -32,6 +33,7 @@ pub(crate) async fn execute( .await? .get_receipt() .await?; + require_successful_receipt("unbond license", &receipt)?; log!( out, "Queued {} FOLD for exit (tx: {:#x})", @@ -71,6 +73,7 @@ pub(crate) async fn execute( .await? .get_receipt() .await?; + require_successful_receipt("claim exits", &receipt)?; log!(out, "Claimed exits (tx: {:#x})", receipt.transaction_hash); } } @@ -91,6 +94,7 @@ async fn bond_license(out: Console, ctx: &ChainContext, amount: &str) -> Result< .await? .get_receipt() .await?; + require_successful_receipt("bond license", &receipt)?; log!( out, "Bonded {} FOLD (tx: {:#x})", diff --git a/crates/cli/src/ciphernode/lifecycle.rs b/crates/cli/src/ciphernode/lifecycle.rs index a855e871ac..14a35f2084 100644 --- a/crates/cli/src/ciphernode/lifecycle.rs +++ b/crates/cli/src/ciphernode/lifecycle.rs @@ -7,6 +7,7 @@ use alloy::primitives::U256; use anyhow::{bail, Result}; use e3_console::{log, Console}; +use e3_utils::require_successful_receipt; use super::context::ChainContext; use super::utils::{format_amount, parse_amount}; @@ -19,6 +20,7 @@ pub(crate) async fn register(out: Console, ctx: &ChainContext) -> Result<()> { .await? .get_receipt() .await?; + require_successful_receipt("register ciphernode", &receipt)?; log!( out, "Registered ciphernode on {} (tx: {:#x})", @@ -36,6 +38,7 @@ pub(crate) async fn deregister(out: Console, ctx: &ChainContext) -> Result<()> { .await? .get_receipt() .await?; + require_successful_receipt("deregister ciphernode", &receipt)?; log!( out, "Deregistration requested (tx: {:#x})", @@ -71,6 +74,7 @@ pub(crate) async fn deactivate( .await? .get_receipt() .await?; + require_successful_receipt("remove ticket balance", &receipt)?; log!( out, "Removed {} tickets (tx: {:#x})", @@ -90,6 +94,7 @@ pub(crate) async fn deactivate( .await? .get_receipt() .await?; + require_successful_receipt("unbond license", &receipt)?; log!( out, "Queued {} FOLD for exit (tx: {:#x})", diff --git a/crates/cli/src/ciphernode/tickets.rs b/crates/cli/src/ciphernode/tickets.rs index 55fbbd2fef..547d112b3e 100644 --- a/crates/cli/src/ciphernode/tickets.rs +++ b/crates/cli/src/ciphernode/tickets.rs @@ -6,6 +6,7 @@ use anyhow::Result; use e3_console::{log, Console}; +use e3_utils::require_successful_receipt; use super::context::ChainContext; use super::utils::{ensure_allowance, parse_amount}; @@ -31,6 +32,7 @@ pub(crate) async fn execute( .await? .get_receipt() .await?; + require_successful_receipt("add ticket balance", &receipt)?; log!( out, "Purchased {} tickets (tx: {:#x})", @@ -49,6 +51,7 @@ pub(crate) async fn execute( .await? .get_receipt() .await?; + require_successful_receipt("remove ticket balance", &receipt)?; log!( out, "Removed {} tickets (tx: {:#x})", diff --git a/crates/cli/src/ciphernode/utils.rs b/crates/cli/src/ciphernode/utils.rs index 3fb77b8fea..aa6f315537 100644 --- a/crates/cli/src/ciphernode/utils.rs +++ b/crates/cli/src/ciphernode/utils.rs @@ -6,8 +6,9 @@ use std::str::FromStr; -use alloy::primitives::{Address, U256}; -use anyhow::{bail, Context, Result}; +use alloy::primitives::{Address, U256}; +use anyhow::{bail, Context, Result}; +use e3_utils::require_successful_receipt; use super::context::ChainContext; @@ -80,11 +81,12 @@ pub(crate) async fn ensure_allowance( return Ok(()); } - erc20 - .approve(spender, amount) - .send() - .await? - .get_receipt() - .await?; - Ok(()) -} + let receipt = erc20 + .approve(spender, amount) + .send() + .await? + .get_receipt() + .await?; + require_successful_receipt("approve token allowance", &receipt)?; + Ok(()) +} diff --git a/crates/cli/src/faucet.rs b/crates/cli/src/faucet.rs index 17b5b2c61f..9771f10deb 100644 --- a/crates/cli/src/faucet.rs +++ b/crates/cli/src/faucet.rs @@ -141,6 +141,7 @@ pub async fn execute(out: Console, config: &AppConfig, selection: Option<&str>) })? .get_receipt() .await?; + e3_utils::require_successful_receipt("request faucet funds", &receipt)?; log!( out, diff --git a/crates/evm-helpers/src/contracts.rs b/crates/evm-helpers/src/contracts.rs index dd4e9d4db5..4515838972 100644 --- a/crates/evm-helpers/src/contracts.rs +++ b/crates/evm-helpers/src/contracts.rs @@ -465,6 +465,7 @@ impl InterfoldWrite for InterfoldContract { let builder = contract.request(e3_request).nonce(nonce); let receipt = builder.send().await?.get_receipt().await?; + e3_utils::require_successful_receipt("request E3", &receipt)?; Ok((receipt, e3_id)) } @@ -479,6 +480,7 @@ impl InterfoldWrite for InterfoldContract { let contract = Interfold::new(self.contract_address, &self.provider); let builder = contract.registerE3Program(e3_program).nonce(nonce); let receipt = builder.send().await?.get_receipt().await?; + e3_utils::require_successful_receipt("register E3 program", &receipt)?; Ok(receipt) } @@ -500,6 +502,7 @@ impl InterfoldWrite for InterfoldContract { .publishCiphertextOutput(e3_id, data, proof) .nonce(nonce); let receipt = builder.send().await?.get_receipt().await?; + e3_utils::require_successful_receipt("publish ciphertext output", &receipt)?; Ok(receipt) } @@ -521,6 +524,7 @@ impl InterfoldWrite for InterfoldContract { .publishPlaintextOutput(e3_id, data, proof) .nonce(nonce); let receipt = builder.send().await?.get_receipt().await?; + e3_utils::require_successful_receipt("publish plaintext output", &receipt)?; Ok(receipt) } diff --git a/crates/evm/src/actors/ciphernode_registry_sol.rs b/crates/evm/src/actors/ciphernode_registry_sol.rs index 64f59a2443..c09cd925a0 100644 --- a/crates/evm/src/actors/ciphernode_registry_sol.rs +++ b/crates/evm/src/actors/ciphernode_registry_sol.rs @@ -22,7 +22,7 @@ use e3_events::{ EType, EffectsEnabled, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, Proof, PublicKeyAggregated, Shutdown, TicketGenerated, TicketId, }; -use e3_utils::{ArcBytes, NotifySync, MAILBOX_LIMIT}; +use e3_utils::{require_successful_receipt, ArcBytes, NotifySync, MAILBOX_LIMIT}; use std::collections::{HashMap, HashSet}; use tracing::{error, info}; @@ -379,6 +379,7 @@ pub async fn submit_ticket_to_registry( let pending = builder.send().await?; drop(_nonce_guard); let receipt = pending.get_receipt().await?; + require_successful_receipt("publish plaintext output", &receipt)?; Ok(receipt) } }, @@ -381,5 +381,6 @@ async fn process_e3_failure( let pending = builder.send().await?; drop(_nonce_guard); let receipt = pending.get_receipt().await?; + require_successful_receipt("process E3 failure", &receipt)?; Ok(receipt) } diff --git a/crates/evm/src/actors/slashing_manager_sol_writer.rs b/crates/evm/src/actors/slashing_manager_sol_writer.rs index 833d9f977c..544984c584 100644 --- a/crates/evm/src/actors/slashing_manager_sol_writer.rs +++ b/crates/evm/src/actors/slashing_manager_sol_writer.rs @@ -30,7 +30,7 @@ use e3_events::InterfoldEvent; use e3_events::InterfoldEventData; use e3_events::Shutdown; use e3_events::{AccusationQuorumReached, EType}; -use e3_utils::NotifySync; +use e3_utils::{require_successful_receipt, NotifySync}; use tracing::{info, warn}; /// Submits `AccusationQuorumReached` events as slash proposals on-chain. @@ -241,6 +241,7 @@ async fn submit_slash_proposal( }; drop(_nonce_guard); let receipt = pending.get_receipt().await?; + require_successful_receipt("submit slashing evidence", &receipt)?; Ok(receipt) } }) diff --git a/crates/utils/src/lib.rs b/crates/utils/src/lib.rs index e7df9f5035..895bc33d6f 100644 --- a/crates/utils/src/lib.rs +++ b/crates/utils/src/lib.rs @@ -15,6 +15,7 @@ pub mod helpers; pub mod path; pub mod retry; pub mod serde_bytes; +pub mod transaction; pub mod utility_types; pub use actix::NotifySync; pub use alloy::*; @@ -27,4 +28,5 @@ pub use helpers::*; pub use path::*; pub use retry::*; pub use serde_bytes::AsBytesSerde; +pub use transaction::require_successful_receipt; pub use utility_types::*; diff --git a/crates/utils/src/transaction.rs b/crates/utils/src/transaction.rs new file mode 100644 index 0000000000..0e4d2a56ef --- /dev/null +++ b/crates/utils/src/transaction.rs @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use alloy::{primitives::B256, rpc::types::TransactionReceipt}; +use std::{error::Error, fmt}; + +/// A transaction was mined but its receipt records an EVM execution failure. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RevertedTransaction { + operation: String, + transaction_hash: B256, +} + +impl fmt::Display for RevertedTransaction { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{} transaction {} reverted on chain", + self.operation, self.transaction_hash + ) + } +} + +impl Error for RevertedTransaction {} + +/// Reject a mined transaction whose consensus receipt reports execution failure. +pub fn require_successful_receipt( + operation: &str, + receipt: &TransactionReceipt, +) -> Result<(), RevertedTransaction> { + require_successful_status(operation, receipt.transaction_hash, receipt.status()) +} + +fn require_successful_status( + operation: &str, + transaction_hash: B256, + succeeded: bool, +) -> Result<(), RevertedTransaction> { + if !succeeded { + return Err(RevertedTransaction { + operation: operation.to_owned(), + transaction_hash, + }); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn successful_status_is_accepted() { + assert!(require_successful_status("publish", B256::ZERO, true).is_ok()); + } + + #[test] + fn failed_status_reports_operation_and_hash() { + let hash = B256::with_last_byte(7); + let error = require_successful_status("publish", hash, false).unwrap_err(); + let message = error.to_string(); + assert!(message.contains("publish")); + assert!(message.contains(&hash.to_string())); + assert!(message.contains("reverted")); + } +} From 6c70941dc0037c7b6e5af7cb795d279585587e64 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 17:01:38 +0500 Subject: [PATCH 31/62] fix(sync): replay aggregates in global timestamp order --- crates/sync/src/actors/sync.rs | 54 +++++++++++++++++++++++++++++++++- 1 file changed, 53 insertions(+), 1 deletion(-) diff --git a/crates/sync/src/actors/sync.rs b/crates/sync/src/actors/sync.rs index bfc32997d5..c20f675594 100644 --- a/crates/sync/src/actors/sync.rs +++ b/crates/sync/src/actors/sync.rs @@ -67,9 +67,10 @@ pub async fn sync( snapshot.to_sequence_map(), addr, ))?; - let events = rx.await?.into_events(); + let mut events = rx.await?.into_events(); info!("{} EventStore events loaded.", events.len()); seed_clock_from_replay(bus, &events)?; + sort_replay_events(&mut events); info!("Replaying events to actors..."); // 4. Replay the EventStore events to all listeners (except effects). @@ -184,6 +185,14 @@ fn seed_clock_from_replay(bus: &BusHandle, events: &[InterfoldEvent]) -> Result< } Ok(()) } + +fn sort_replay_events(events: &mut [InterfoldEvent]) { + // EventStoreRouter gathers one query response per aggregate. Those actor responses can arrive + // in any order, so replay must re-establish the global HLC order before stateful subscribers + // observe cross-aggregate dependencies. + events.sort_by_key(|event| event.ts()); +} + /// Validate or initialize the durable schema marker before runtime actors can write state. /// /// A fresh empty store is stamped synchronously; incompatible or unversioned non-empty state @@ -296,6 +305,7 @@ impl SnapshotLoaded { mod tests { use super::{ collect_historical_evm_events, preflight_schema_version, seed_clock_from_replay, + sort_replay_events, }; use crate::domain::SyncPlanner; use e3_ciphernode_builder::EventSystem; @@ -486,6 +496,48 @@ mod tests { assert!(HlcTimestamp::from(bus.ts()?) > durable); Ok(()) } + + #[actix::test] + async fn replay_restores_global_timestamp_order_across_aggregates() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-ordered-sync-replay"); + let history = bus.history(); + let mut events = vec![ + InterfoldEvent::::test_event("third") + .id(3) + .aggregate_id(3) + .ts(30) + .seq(1) + .build(), + InterfoldEvent::::test_event("first") + .id(1) + .aggregate_id(1) + .ts(10) + .seq(1) + .build(), + InterfoldEvent::::test_event("second") + .id(2) + .aggregate_id(2) + .ts(20) + .seq(1) + .build(), + ]; + + sort_replay_events(&mut events); + for event in events { + bus.event_bus().try_send(event)?; + } + + let received = history.send(TakeEvents::new(3)).await?; + let timestamps = received + .events + .iter() + .map(|event| event.ts()) + .collect::>(); + assert_eq!(timestamps, vec![10, 20, 30]); + Ok(()) + } + /// Verify that `run_once::` correctly gates event subscriptions. /// /// Simulates the sync flow: From ffba237e0efbcbdec10e6f1d03e9ca38692f0c5c Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 17:05:34 +0500 Subject: [PATCH 32/62] fix(net): bind DHT metadata to document payloads --- crates/net/src/actors/document_publisher.rs | 93 +++++++++++++++-- crates/net/src/domain/event_conversion.rs | 110 +++++++++++++++++++- 2 files changed, 187 insertions(+), 16 deletions(-) diff --git a/crates/net/src/actors/document_publisher.rs b/crates/net/src/actors/document_publisher.rs index 7ac2235e61..af44dfa15a 100644 --- a/crates/net/src/actors/document_publisher.rs +++ b/crates/net/src/actors/document_publisher.rs @@ -292,6 +292,11 @@ pub async fn handle_document_published_notification( ) .await?; + // The gossiped metadata is not covered by the DHT content hash. Bind it to the decoded + // payload before persisting DocumentReceived; otherwise a notification for an E3 this node is + // interested in can inject a content-addressed document for a different E3 or party route. + EventConversionService::validate_received(&event.meta, &value)?; + debug!("Sending received event..."); bus.publish_from_remote( DocumentReceived { @@ -444,7 +449,7 @@ impl EventConverter { /// Note: Filtering already happened in DocumentPublisher before DHT fetch. fn handle_document_received(&self, msg: TypedEvent) -> Result<()> { let (msg, ctx) = msg.into_components(); - match EventConversionService::decode_received(&msg.value.extract_bytes())? { + match EventConversionService::decode_received(&msg.meta, &msg.value)? { IncomingDocument::ThresholdShare(evt) => { self.bus.publish(evt, ctx)?; } @@ -556,7 +561,7 @@ mod tests { use chrono::Utc; use e3_ciphernode_builder::EventSystem; use e3_events::{ - BusHandle, CiphernodeSelected, DocumentKind, DocumentMeta, E3id, GetEvents, + BusHandle, CiphernodeSelected, DocumentKind, DocumentMeta, E3id, EncryptionKey, GetEvents, HistoryCollector, InterfoldError, InterfoldEvent, PublishDocumentRequested, TakeEvents, }; use libp2p::kad::{GetRecordError, PutRecordError, RecordKey}; @@ -829,9 +834,15 @@ mod tests { let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, history, _, _) = setup_test()?; - let value = ArcBytes::from_bytes(b"I am a special document"); let expires_at = Utc::now() + chrono::Duration::days(1); let e3_id = E3id::new("1243", 1); + let value = EventConversionService::encryption_key_to_request(EncryptionKeyCreated { + e3_id: e3_id.clone(), + key: Arc::new(EncryptionKey::new(1, ArcBytes::from_bytes(b"public key"))), + external: false, + })? + .expect("local key should produce a document") + .value; let cid = ContentHash::from_content(&value); // 1. Ensure the publisher is interested in the id by receiving CiphernodeSelected @@ -887,7 +898,7 @@ mod tests { net_evt_tx.send(NetEvent::DhtGetRecordSucceeded { key: cid, correlation_id, - value, // This will error because this is not a ReceivableDocument which is fine + value: value.clone(), })?; // wait for events to settle @@ -896,25 +907,85 @@ mod tests { // Check event was dispatched let events = history.send(GetEvents::new()).await?; let Some(InterfoldEventData::DocumentReceived(DocumentReceived { value: doc, .. })) = - events - .iter() - // Filter out the error - .filter(|e| e.event_type() != "InterfoldError") - .last() - .map(|e| e.get_data()) + events.iter().find_map(|event| match event.get_data() { + data @ InterfoldEventData::DocumentReceived(_) => Some(data), + _ => None, + }) else { bail!("No event sent"); }; assert_eq!( doc.extract_bytes(), - b"I am a special document", + value.extract_bytes(), "document did not match" ); Ok(()) } + #[actix::test] + async fn notification_cannot_relabel_payload_for_another_e3() -> Result<()> { + let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _rx, history, errors, _) = + setup_test()?; + let interested_e3 = E3id::new("100", 1); + let payload_e3 = E3id::new("200", 1); + let value = EventConversionService::encryption_key_to_request(EncryptionKeyCreated { + e3_id: payload_e3, + key: Arc::new(EncryptionKey::new(1, ArcBytes::from_bytes(b"public key"))), + external: false, + })? + .expect("local key should produce a document") + .value; + let key = ContentHash::from_content(&value); + + bus.publish_without_context(CiphernodeSelected { + e3_id: interested_e3.clone(), + threshold_m: 3, + threshold_n: 5, + ..CiphernodeSelected::default() + })?; + net_evt_tx.send(NetEvent::GossipData( + GossipData::DocumentPublishedNotification(DocumentPublishedNotification { + key: key.clone(), + meta: DocumentMeta::new( + interested_e3, + DocumentKind::TrBFV, + vec![], + Some(Utc::now() + chrono::Duration::days(1)), + ), + ts: 100, + }), + ))?; + + let Some(NetCommand::DhtGetRecord { correlation_id, .. }) = + timeout(Duration::from_secs(1), net_cmd_rx.recv()) + .await + .expect("did not receive DhtGetRecord") + else { + bail!("msg not as expected"); + }; + net_evt_tx.send(NetEvent::DhtGetRecordSucceeded { + key, + correlation_id, + value, + })?; + + let error_events = errors.send(TakeEvents::new(1)).await?; + let error: InterfoldError = error_events.events.first().unwrap().try_into()?; + assert!(error.message.contains("metadata E3 1:100")); + assert!(error.message.contains("payload E3 1:200")); + + let events = history.send(GetEvents::new()).await?; + assert!( + !events + .iter() + .any(|event| matches!(event.get_data(), InterfoldEventData::DocumentReceived(_))), + "mismatched document must not be persisted" + ); + Ok(()) + } + pub fn is_between(instant: Instant, start: Instant, end: Instant) -> bool { let (min, max) = if start <= end { (start, end) diff --git a/crates/net/src/domain/event_conversion.rs b/crates/net/src/domain/event_conversion.rs index 450607a67b..4d921c6f62 100644 --- a/crates/net/src/domain/event_conversion.rs +++ b/crates/net/src/domain/event_conversion.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::{Context, Result}; +use anyhow::{ensure, Context, Result}; use e3_events::{ DecryptionKeyShared, DocumentKind, DocumentMeta, EncryptionKeyCreated, EncryptionKeyReceived, Filter, PublishDocumentRequested, ThresholdShareCreated, @@ -34,6 +34,44 @@ impl ReceivableDocument { pub fn from_bytes(bytes: &[u8]) -> Result { decode(bytes, MAX_DHT_DOCUMENT_BYTES) } + + fn e3_id(&self) -> &e3_events::E3id { + match self { + Self::ThresholdShareCreated(event) => &event.e3_id, + Self::EncryptionKeyCreated(event) => &event.e3_id, + Self::DecryptionKeyShared(event) => &event.e3_id, + } + } + + fn validate_meta(&self, meta: &DocumentMeta) -> Result<()> { + ensure!( + self.e3_id() == &meta.e3_id, + "DHT document metadata E3 {} does not match payload E3 {}", + meta.e3_id, + self.e3_id() + ); + ensure!( + matches!(&meta.kind, DocumentKind::TrBFV), + "DHT document metadata has an unsupported document kind" + ); + + match self { + Self::ThresholdShareCreated(event) => ensure!( + matches!( + meta.filter.as_slice(), + [Filter::Item(target)] if *target == event.target_party_id + ), + "threshold-share metadata must target payload party {}", + event.target_party_id + ), + Self::EncryptionKeyCreated(_) | Self::DecryptionKeyShared(_) => ensure!( + meta.filter.is_empty(), + "broadcast key document metadata must not contain party filters" + ), + } + + Ok(()) + } } /// A document received from the network, decoded into the internal event it should be @@ -106,12 +144,18 @@ impl EventConversionService { Ok(Some(PublishDocumentRequested::new(meta, value))) } + /// Validate that independently-gossiped metadata describes the content-addressed DHT payload. + pub fn validate_received(meta: &DocumentMeta, bytes: &[u8]) -> Result<()> { + let receivable = Self::decode_and_validate(meta, bytes)?; + drop(receivable); + Ok(()) + } + /// Decode a received document payload into the internal event that should be published. /// /// Note: party filtering already happened in `DocumentPublisher` before the DHT fetch. - pub fn decode_received(bytes: &[u8]) -> Result { - let receivable = ReceivableDocument::from_bytes(bytes) - .context("Could not deserialize document bytes")?; + pub fn decode_received(meta: &DocumentMeta, bytes: &[u8]) -> Result { + let receivable = Self::decode_and_validate(meta, bytes)?; Ok(match receivable { ReceivableDocument::ThresholdShareCreated(evt) => { debug!( @@ -148,6 +192,13 @@ impl EventConversionService { } }) } + + fn decode_and_validate(meta: &DocumentMeta, bytes: &[u8]) -> Result { + let receivable = ReceivableDocument::from_bytes(bytes) + .context("Could not deserialize document bytes")?; + receivable.validate_meta(meta)?; + Ok(receivable) + } } fn encode(doc: &ReceivableDocument) -> Result { @@ -157,9 +208,58 @@ fn encode(doc: &ReceivableDocument) -> Result { #[cfg(test)] mod tests { use super::*; + use e3_events::{E3id, EncryptionKey}; + use std::sync::Arc; + + fn encryption_key_document(e3_id: E3id) -> ReceivableDocument { + ReceivableDocument::EncryptionKeyCreated(EncryptionKeyCreated { + e3_id, + key: Arc::new(EncryptionKey::new(1, ArcBytes::from_bytes(b"pk"))), + external: false, + }) + } + + fn meta(e3_id: E3id) -> DocumentMeta { + DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], None) + } #[test] fn decode_received_rejects_garbage() { - assert!(EventConversionService::decode_received(b"not a document").is_err()); + assert!(EventConversionService::decode_received( + &meta(E3id::new("1", 1)), + b"not a document" + ) + .is_err()); + } + + #[test] + fn received_document_must_match_metadata_e3_and_chain() { + let payload_id = E3id::new("9", 1); + let bytes = encryption_key_document(payload_id.clone()) + .to_bytes() + .unwrap(); + + EventConversionService::validate_received(&meta(payload_id), &bytes).unwrap(); + + let wrong_e3 = EventConversionService::validate_received(&meta(E3id::new("10", 1)), &bytes) + .unwrap_err(); + assert!(wrong_e3.to_string().contains("1:10")); + assert!(wrong_e3.to_string().contains("1:9")); + + let wrong_chain = + EventConversionService::validate_received(&meta(E3id::new("9", 2)), &bytes) + .unwrap_err(); + assert!(wrong_chain.to_string().contains("2:9")); + assert!(wrong_chain.to_string().contains("1:9")); + } + + #[test] + fn broadcast_key_document_rejects_party_filter_relabeling() { + let e3_id = E3id::new("9", 1); + let bytes = encryption_key_document(e3_id.clone()).to_bytes().unwrap(); + let filtered = DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![Filter::Item(3)], None); + + let error = EventConversionService::validate_received(&filtered, &bytes).unwrap_err(); + assert!(error.to_string().contains("must not contain party filters")); } } From 9d1ae226a6b7fcd64e5a5fc3b0a18def5c5244fc Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 17:06:38 +0500 Subject: [PATCH 33/62] fix(net): backpressure remote actor ingress --- crates/net/src/actors/document_publisher.rs | 8 +++++++- crates/net/src/actors/net_event_translator.rs | 5 ++++- crates/net/src/actors/net_sync_manager.rs | 14 +++++++++----- 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/crates/net/src/actors/document_publisher.rs b/crates/net/src/actors/document_publisher.rs index af44dfa15a..8c529eaf6a 100644 --- a/crates/net/src/actors/document_publisher.rs +++ b/crates/net/src/actors/document_publisher.rs @@ -109,7 +109,13 @@ impl DocumentPublisher { if let NetEvent::GossipData(GossipData::DocumentPublishedNotification(data)) = event { - addr.do_send(data) + if let Err(error) = addr.send(data).await { + tracing::warn!( + %error, + "DocumentPublisher stopped; ending DHT notification ingress" + ); + break; + } } } } diff --git a/crates/net/src/actors/net_event_translator.rs b/crates/net/src/actors/net_event_translator.rs index fd002e9ccc..5b162f994d 100644 --- a/crates/net/src/actors/net_event_translator.rs +++ b/crates/net/src/actors/net_event_translator.rs @@ -66,7 +66,10 @@ impl NetEventTranslator { while let Some(event) = super::recv_net_event(&mut rx, "NetEventTranslator").await { if let NetEvent::GossipData(data) = event { if let GossipData::GossipBytes(_) = data { - addr.do_send(LibP2pEvent(data)); + if let Err(error) = addr.send(LibP2pEvent(data)).await { + warn!(%error, "NetEventTranslator stopped; ending gossip ingress"); + break; + } } } } diff --git a/crates/net/src/actors/net_sync_manager.rs b/crates/net/src/actors/net_sync_manager.rs index 9f4a8ca60d..931bbb43d3 100644 --- a/crates/net/src/actors/net_sync_manager.rs +++ b/crates/net/src/actors/net_sync_manager.rs @@ -297,14 +297,18 @@ impl NetSyncManager { async move { while let Some(event) = super::recv_net_event(&mut events, "NetSyncManager").await { debug!("Received event {:?}", event); - match event { + let delivery = match event { // Someone is asking for our sync - NetEvent::IncomingRequest(value) => addr.do_send(value), + NetEvent::IncomingRequest(value) => addr.send(value).await, NetEvent::AllPeersDialed { connected, total } => { - addr.do_send(AllPeersDialed { connected, total }) + addr.send(AllPeersDialed { connected, total }).await } - NetEvent::ConnectionEstablished { .. } => addr.do_send(PeerConnected), - _ => (), + NetEvent::ConnectionEstablished { .. } => addr.send(PeerConnected).await, + _ => continue, + }; + if let Err(error) = delivery { + warn!(%error, "NetSyncManager stopped; ending sync ingress"); + break; } } } From 75e9f0e08897939e927710257908f29af98920f1 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 17:11:21 +0500 Subject: [PATCH 34/62] fix(slashing): defer and coalesce replayed submissions --- .../flow-trace/05_FAILURE_REFUND_SLASHING.md | 24 ++- .../src/actors/slashing_manager_sol_writer.rs | 107 ++++++++++- crates/evm/src/domain/slash_submission.rs | 174 +++++++++++++++++- 3 files changed, 287 insertions(+), 18 deletions(-) diff --git a/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md b/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md index 24dc382ddb..7718d8d8d7 100644 --- a/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md +++ b/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md @@ -434,7 +434,12 @@ AccusationQuorumReached event arrives at SlashingManagerSolWriter │ ├─ Only for SLASHABLE outcomes (AccusedFaulted, Equivocation): │ -├─ 1. STAGGERED SUBMISSION (fallback submitters): +├─ 1. EFFECT AND REPLAY GATE: +│ Before EffectsEnabled (startup replay), retain the intent without sending a transaction +│ Coalesce by the contract replay tuple (chainId, e3Id, accused, proofType) +│ After EffectsEnabled, release each retained intent once and track it in flight +│ +├─ 2. STAGGERED SUBMISSION (fallback submitters): │ Rank all agreeing voters by address (sorted ascending) │ My rank = position in sorted list │ │ @@ -445,21 +450,24 @@ AccusationQuorumReached event arrives at SlashingManagerSolWriter │ → Prevents multiple nodes wasting gas on same slash │ → Higher-rank submitters expect DuplicateEvidence revert │ -├─ 2. Encode attestation evidence: +├─ 3. Encode attestation evidence: │ proof = abi.encode( │ proofType, // uint256 — which proof failed (C0-C7) │ voters[], // address[] — sorted ascending -│ agrees[], // bool[] — all true (only agreeing votes submitted) │ dataHashes[], // bytes32[] — per-voter data hashes +│ evidence, // bytes — shared evidence preimage +│ deadline, // uint256 — common signed deadline │ signatures[] // bytes[] — per-voter ECDSA signatures │ ) │ -├─ 3. Call SlashingManager.proposeSlash(e3Id, accused, proof) +├─ 4. Prefer proposeSlashByDkgParty(e3Id, partyId, proof) when the +│ canonical DKG slot resolves; otherwise call proposeSlash(e3Id, accused, proof) │ → On-chain verification happens (see Lane A below) │ -└─ 4. Handle result: +└─ 5. Handle result: ├─ Success: log transaction hash - └─ DuplicateEvidence: expected for fallback submitters (logged as warning) + ├─ DuplicateEvidence / stale committee attribution: terminal and logged as warning + └─ Other RPC or contract failures: reported and made eligible for a later retry event ``` ### Lane A: Attestation-Based Slashing (Permissionless, Atomic) @@ -961,7 +969,9 @@ Slash Reasons (derived from ProofType for Lane A): │ └─ AccusationQuorumReached event published │ │ │ │ 7. ON-CHAIN SUBMISSION (SlashingManagerSolWriter) │ -│ ├─ Staggered: rank 0 submits immediately │ +│ ├─ Defers replayed intents until EffectsEnabled │ +│ ├─ Coalesces the contract replay tuple while in flight │ +│ ├─ Staggered: rank 0 submits immediately │ │ │ ranks 1+ wait rank×30s as fallback │ │ ├─ Encodes attestation evidence (votes + signatures) │ │ └─ Calls SlashingManager.proposeSlash(e3Id, operator, proof)│ diff --git a/crates/evm/src/actors/slashing_manager_sol_writer.rs b/crates/evm/src/actors/slashing_manager_sol_writer.rs index 544984c584..94da6954f1 100644 --- a/crates/evm/src/actors/slashing_manager_sol_writer.rs +++ b/crates/evm/src/actors/slashing_manager_sol_writer.rs @@ -12,7 +12,10 @@ use crate::contracts::{ICiphernodeRegistry, ISlashingManager}; use crate::domain::attestation_evidence::encode_attestation_evidence; use crate::domain::error_decoder::format_evm_error; -use crate::domain::slash_submission::{should_submit_slash, submission_delay, submission_rank}; +use crate::domain::slash_submission::{ + should_submit_slash, submission_delay, submission_rank, SlashIntentKey, + SlashSubmissionDecision, SlashSubmissionGate, +}; use crate::helpers::{transaction_nonce_guard, EthProvider}; use crate::send_tx_with_retry; use actix::prelude::*; @@ -30,7 +33,7 @@ use e3_events::InterfoldEvent; use e3_events::InterfoldEventData; use e3_events::Shutdown; use e3_events::{AccusationQuorumReached, EType}; -use e3_utils::{require_successful_receipt, NotifySync}; +use e3_utils::{require_successful_receipt, NotifySync, MAILBOX_LIMIT}; use tracing::{info, warn}; /// Submits `AccusationQuorumReached` events as slash proposals on-chain. @@ -38,6 +41,21 @@ pub struct SlashingManagerSolWriter

{ provider: EthProvider

, contract_address: Address, bus: BusHandle, + submissions: SlashSubmissionGate, +} + +#[derive(Message)] +#[rtype(result = "()")] +struct SubmitSlashIntent { + key: SlashIntentKey, + event: AccusationQuorumReached, +} + +#[derive(Message)] +#[rtype(result = "()")] +struct SlashSubmissionFinished { + key: SlashIntentKey, + terminal: bool, } impl SlashingManagerSolWriter

{ @@ -50,6 +68,7 @@ impl SlashingManagerSolWriter

provider, contract_address, bus: bus.clone(), + submissions: SlashSubmissionGate::new(), }) } @@ -60,7 +79,11 @@ impl SlashingManagerSolWriter

) -> Result>> { let addr = SlashingManagerSolWriter::new(bus, provider, contract_address)?.start(); bus.subscribe_all( - &[EventType::AccusationQuorumReached, EventType::Shutdown], + &[ + EventType::AccusationQuorumReached, + EventType::EffectsEnabled, + EventType::Shutdown, + ], addr.clone().into(), ); Ok(addr) @@ -69,6 +92,10 @@ impl SlashingManagerSolWriter

impl Actor for SlashingManagerSolWriter

{ type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } } impl Handler @@ -96,7 +123,51 @@ impl Handler &data.outcome, rank, ) { - ctx.notify(data); + if encode_attestation_evidence(&data).is_none() { + self.bus.err( + EType::Evm, + anyhow::anyhow!( + "Refusing malformed slash intent for E3 {}: votes or evidence are empty", + data.e3_id + ), + ); + return; + } + match self.submissions.admit(data.clone()) { + Ok((key, SlashSubmissionDecision::Submit)) => { + ctx.notify(SubmitSlashIntent { key, event: data }); + } + Ok((_, SlashSubmissionDecision::Defer)) => { + info!(e3_id = %data.e3_id, "Deferred slash intent until effects are enabled"); + } + Ok((_, SlashSubmissionDecision::IgnoreDuplicate)) => { + info!(e3_id = %data.e3_id, "Ignored duplicate slash intent"); + } + Err(error) => self.bus.err(EType::Evm, error), + } + } + } + InterfoldEventData::EffectsEnabled(_) => { + let deferred = self.submissions.enable_effects(); + if !deferred.is_empty() { + info!( + intents = deferred.len(), + "Releasing deferred slash intents after startup reconciliation" + ); + let address = ctx.address(); + ctx.spawn( + async move { + for (key, event) in deferred { + if let Err(error) = + address.send(SubmitSlashIntent { key, event }).await + { + warn!(%error, "Slashing writer stopped with deferred intents pending"); + break; + } + } + } + .into_actor(self), + ); } } InterfoldEventData::Shutdown(data) => self.notify_sync(ctx, data), @@ -105,18 +176,20 @@ impl Handler } } -impl Handler +impl Handler for SlashingManagerSolWriter

{ type Result = ResponseFuture<()>; - fn handle(&mut self, msg: AccusationQuorumReached, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: SubmitSlashIntent, ctx: &mut Self::Context) -> Self::Result { Box::pin({ let contract_address = self.contract_address; let provider = self.provider.clone(); let bus = self.bus.clone(); let my_addr = self.provider.provider().default_signer_address(); + let address = ctx.address(); async move { + let SubmitSlashIntent { key, event: msg } = msg; // Compute this node's submission rank for staggered fallback let rank = submission_rank(msg.votes_for.iter().map(|v| v.voter), my_addr).unwrap_or(0); @@ -132,16 +205,17 @@ impl Handler { info!(tx=%receipt.transaction_hash, "Submitted attestation-based slash proposal on-chain"); + true } Err(err) => { let decoded = format_evm_error(&err); let benign = decoded.contains("OperatorNotInCommittee") || decoded.contains("VoterNotInCommittee") || decoded.contains("DuplicateEvidence"); - if rank > 0 || benign { + if benign { // Fallback submitters expect DuplicateEvidence reverts // when the primary submitter has already landed the tx. // Operator/VoterNotInCommittee indicate a stale off-chain accusation @@ -153,13 +227,30 @@ impl Handler Handler + for SlashingManagerSolWriter

+{ + type Result = (); + + fn handle(&mut self, msg: SlashSubmissionFinished, _: &mut Self::Context) -> Self::Result { + self.submissions.finish(&msg.key, msg.terminal); + } +} + impl Handler for SlashingManagerSolWriter

{ diff --git a/crates/evm/src/domain/slash_submission.rs b/crates/evm/src/domain/slash_submission.rs index 24aff27813..42dfd07a2c 100644 --- a/crates/evm/src/domain/slash_submission.rs +++ b/crates/evm/src/domain/slash_submission.rs @@ -12,10 +12,14 @@ //! `rank * SUBMITTER_DELAY_SECS` so on-chain `DuplicateEvidence` protection lets //! at most one slash execute. -use std::time::Duration; +use std::{ + collections::{BTreeMap, BTreeSet}, + time::Duration, +}; -use alloy::primitives::Address; -use e3_events::AccusationOutcome; +use alloy::primitives::{Address, U256}; +use anyhow::{Context, Result}; +use e3_events::{AccusationOutcome, AccusationQuorumReached}; /// Maximum number of voters eligible to attempt on-chain submission. /// Rank 0 submits immediately, rank 1 after one delay interval, etc. @@ -25,6 +29,103 @@ pub(crate) const MAX_SLASH_SUBMITTERS: usize = 3; /// Rank N waits N * SUBMITTER_DELAY_SECS before submitting. pub(crate) const SUBMITTER_DELAY_SECS: u64 = 30; +/// The exact semantic replay domain consumed by `SlashingManager._proposeSlash`. +/// Vote ordering and signatures are deliberately excluded: Solidity permits one +/// submission for `(chain, E3, operator, proof type)` regardless of evidence encoding. +#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub(crate) struct SlashIntentKey { + chain_id: u64, + e3_id: U256, + operator: Address, + proof_type: u8, +} + +impl SlashIntentKey { + pub(crate) fn from_quorum(event: &AccusationQuorumReached) -> Result { + Ok(Self { + chain_id: event.e3_id.chain_id(), + e3_id: event + .e3_id + .clone() + .try_into() + .context("slash intent has a non-numeric E3 id")?, + operator: event.accused, + proof_type: event.proof_type as u8, + }) + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum SlashSubmissionDecision { + Defer, + Submit, + IgnoreDuplicate, +} + +/// Process-local outbox gate for slash submissions. +/// +/// Replayed intents are retained until `EffectsEnabled`, and semantically equivalent +/// events are coalesced while deferred, in flight, or already completed. This prevents +/// startup reconciliation from producing transactions and prevents same-process gas +/// loss from reordered-but-equivalent quorum payloads. +#[derive(Default)] +pub(crate) struct SlashSubmissionGate { + effects_enabled: bool, + deferred: BTreeMap, + in_flight: BTreeSet, + completed: BTreeSet, +} + +impl SlashSubmissionGate { + pub(crate) fn new() -> Self { + Self::default() + } + + pub(crate) fn admit( + &mut self, + event: AccusationQuorumReached, + ) -> Result<(SlashIntentKey, SlashSubmissionDecision)> { + let key = SlashIntentKey::from_quorum(&event)?; + if self.completed.contains(&key) + || self.in_flight.contains(&key) + || self.deferred.contains_key(&key) + { + return Ok((key, SlashSubmissionDecision::IgnoreDuplicate)); + } + + if !self.effects_enabled { + self.deferred.insert(key.clone(), event); + return Ok((key, SlashSubmissionDecision::Defer)); + } + + self.in_flight.insert(key.clone()); + Ok((key, SlashSubmissionDecision::Submit)) + } + + pub(crate) fn enable_effects(&mut self) -> Vec<(SlashIntentKey, AccusationQuorumReached)> { + self.effects_enabled = true; + let deferred = std::mem::take(&mut self.deferred); + deferred + .into_iter() + .filter_map(|(key, event)| { + if self.completed.contains(&key) || self.in_flight.contains(&key) { + None + } else { + self.in_flight.insert(key.clone()); + Some((key, event)) + } + }) + .collect() + } + + pub(crate) fn finish(&mut self, key: &SlashIntentKey, terminal: bool) { + self.in_flight.remove(key); + if terminal { + self.completed.insert(key.clone()); + } + } +} + /// Determine this node's submission rank: its position in the voter set after /// sorting ascending by address. `None` when this node is not among the voters. pub(crate) fn submission_rank(voters: I, my_addr: Address) -> Option @@ -61,6 +162,32 @@ pub(crate) fn submission_delay(rank: usize) -> Duration { #[cfg(test)] mod tests { use super::*; + use alloy::primitives::{Bytes, B256}; + use e3_events::{AccusationQuorumReached, AccusationVote, E3id, ProofType}; + use e3_utils::ArcBytes; + + fn vote(voter: Address) -> AccusationVote { + AccusationVote { + e3_id: E3id::new("1", 1), + accusation_id: B256::ZERO.0, + voter, + data_hash: B256::repeat_byte(7).0, + deadline: 100, + signature: ArcBytes::from_bytes(b"signature"), + } + } + + fn quorum(voters: Vec

) -> AccusationQuorumReached { + AccusationQuorumReached { + e3_id: E3id::new("1", 1), + accuser: Address::repeat_byte(9), + accused: Address::repeat_byte(8), + proof_type: ProofType::C0PkBfv, + votes_for: voters.into_iter().map(vote).collect(), + outcome: AccusationOutcome::AccusedFaulted, + evidence: Bytes::from_static(b"evidence"), + } + } #[test] fn test_submission_rank_sorts_ascending() { @@ -122,4 +249,45 @@ mod tests { Duration::from_secs(2 * SUBMITTER_DELAY_SECS) ); } + + #[test] + fn replayed_submission_is_deferred_and_released_once_after_effects() { + let mut gate = SlashSubmissionGate::new(); + let event = quorum(vec![Address::repeat_byte(1)]); + let (_, decision) = gate.admit(event.clone()).unwrap(); + assert_eq!(decision, SlashSubmissionDecision::Defer); + + let released = gate.enable_effects(); + assert_eq!(released.len(), 1); + assert!(gate.enable_effects().is_empty()); + + let (_, duplicate) = gate.admit(event).unwrap(); + assert_eq!(duplicate, SlashSubmissionDecision::IgnoreDuplicate); + } + + #[test] + fn reordered_votes_share_the_contract_replay_key() { + let a = Address::repeat_byte(1); + let b = Address::repeat_byte(2); + let first = SlashIntentKey::from_quorum(&quorum(vec![a, b])).unwrap(); + let reordered = SlashIntentKey::from_quorum(&quorum(vec![b, a])).unwrap(); + assert_eq!(first, reordered); + } + + #[test] + fn retryable_failure_clears_in_flight_but_terminal_result_does_not() { + let event = quorum(vec![Address::repeat_byte(1)]); + let mut gate = SlashSubmissionGate::new(); + gate.enable_effects(); + + let (key, first) = gate.admit(event.clone()).unwrap(); + assert_eq!(first, SlashSubmissionDecision::Submit); + gate.finish(&key, false); + let (key, retry) = gate.admit(event.clone()).unwrap(); + assert_eq!(retry, SlashSubmissionDecision::Submit); + + gate.finish(&key, true); + let (_, completed) = gate.admit(event).unwrap(); + assert_eq!(completed, SlashSubmissionDecision::IgnoreDuplicate); + } } From 5486d5b8990033e851d5f660c9e30159927c8471 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 17:19:14 +0500 Subject: [PATCH 35/62] fix(program-server): require authenticated compute requests --- crates/program-server/src/lib.rs | 283 ++++++++++++++++++++++--- crates/support-scripts/dev/src/main.rs | 10 +- 2 files changed, 261 insertions(+), 32 deletions(-) diff --git a/crates/program-server/src/lib.rs b/crates/program-server/src/lib.rs index f73bd279a8..a57d8490b6 100644 --- a/crates/program-server/src/lib.rs +++ b/crates/program-server/src/lib.rs @@ -6,11 +6,15 @@ mod types; -use actix_web::{middleware::Logger, web, App, HttpResponse, HttpServer, Result as ActixResult}; -use anyhow::Result; +use actix_web::{ + http::header, middleware::Logger, web, App, HttpRequest, HttpResponse, HttpServer, + Result as ActixResult, +}; +use anyhow::{Context, Result}; use e3_compute_provider::FHEInputs; use serde::Serialize; -use std::{future::Future, pin::Pin, sync::Arc}; +use std::{future::Future, pin::Pin, sync::Arc, time::Duration}; +use tokio::sync::Semaphore; use types::{ComputeRequest, WebhookPayload}; #[derive(Serialize, Debug)] @@ -28,6 +32,9 @@ pub struct E3ProgramServerBuilder { port: Option, host: Option, localhost_rewrite: Option, + bearer_token: Option, + callback_origin: Option, + max_concurrent_jobs: usize, } impl E3ProgramServerBuilder { @@ -42,6 +49,9 @@ impl E3ProgramServerBuilder { port: None, host: None, localhost_rewrite: None, + bearer_token: None, + callback_origin: None, + max_concurrent_jobs: 1, } } @@ -63,14 +73,60 @@ impl E3ProgramServerBuilder { self } + /// Require this bearer token on every compute request. + pub fn with_bearer_token(mut self, token: impl Into) -> Self { + self.bearer_token = Some(token.into()); + self + } + + /// Allow callbacks only to this exact URL origin (scheme, host, and port). + pub fn with_callback_origin(mut self, origin: impl Into) -> Self { + self.callback_origin = Some(origin.into()); + self + } + + /// Bound the number of computations that may execute concurrently. + pub fn with_max_concurrent_jobs(mut self, max_concurrent_jobs: usize) -> Self { + self.max_concurrent_jobs = max_concurrent_jobs; + self + } + /// Build the E3ProgramServer - pub fn build(self) -> E3ProgramServer { - E3ProgramServer { + pub fn build(self) -> Result { + let bearer_token = self + .bearer_token + .filter(|token| !token.is_empty()) + .context("program server requires a non-empty bearer token")?; + let callback_origin = self + .callback_origin + .context("program server requires an allowed callback origin")?; + let callback_origin = parse_http_url(&callback_origin, "callback origin")?; + anyhow::ensure!( + callback_origin.path() == "/" + && callback_origin.query().is_none() + && callback_origin.fragment().is_none(), + "callback origin must not contain a path, query, or fragment" + ); + anyhow::ensure!( + self.max_concurrent_jobs > 0, + "max concurrent jobs must be greater than zero" + ); + let webhook_client = reqwest::Client::builder() + .connect_timeout(Duration::from_secs(5)) + .timeout(Duration::from_secs(30)) + .redirect(reqwest::redirect::Policy::none()) + .build() + .context("failed to build webhook client")?; + Ok(E3ProgramServer { runner: self.runner, port: self.port.unwrap_or(13151), host: self.host.unwrap_or_else(|| "0.0.0.0".to_string()), localhost_rewrite: self.localhost_rewrite, - } + bearer_token: Arc::from(bearer_token), + callback_origin, + webhook_client, + jobs: Arc::new(Semaphore::new(self.max_concurrent_jobs)), + }) } } @@ -80,6 +136,10 @@ pub struct E3ProgramServer { port: u16, host: String, localhost_rewrite: Option, + bearer_token: Arc, + callback_origin: reqwest::Url, + webhook_client: reqwest::Client, + jobs: Arc, } impl E3ProgramServer { @@ -113,6 +173,10 @@ impl E3ProgramServer { let config = AppConfig { runner: Arc::clone(&self.runner), localhost_rewrite: self.localhost_rewrite.clone(), + bearer_token: Arc::clone(&self.bearer_token), + callback_origin: self.callback_origin.clone(), + webhook_client: self.webhook_client.clone(), + jobs: Arc::clone(&self.jobs), }; let server = HttpServer::new(move || { App::new() @@ -134,9 +198,82 @@ impl E3ProgramServer { pub struct AppConfig { pub runner: Arc, pub localhost_rewrite: Option, + bearer_token: Arc, + callback_origin: reqwest::Url, + webhook_client: reqwest::Client, + jobs: Arc, +} + +fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { + let mut difference = left.len() ^ right.len(); + let max_len = left.len().max(right.len()); + for index in 0..max_len { + difference |= left.get(index).copied().unwrap_or(0) as usize + ^ right.get(index).copied().unwrap_or(0) as usize; + } + difference == 0 +} + +fn authorize_compute(request: &HttpRequest, expected_token: &str) -> ActixResult<()> { + let supplied = request + .headers() + .get(header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.strip_prefix("Bearer ")); + if supplied.is_some_and(|token| constant_time_eq(token.as_bytes(), expected_token.as_bytes())) { + Ok(()) + } else { + Err(actix_web::error::ErrorUnauthorized( + "missing or invalid bearer token", + )) + } +} + +fn parse_http_url(value: &str, label: &str) -> Result { + let url = reqwest::Url::parse(value).with_context(|| format!("invalid {label}"))?; + anyhow::ensure!( + matches!(url.scheme(), "http" | "https"), + "{label} must use http or https" + ); + anyhow::ensure!(url.host_str().is_some(), "{label} must contain a host"); + anyhow::ensure!( + url.username().is_empty() && url.password().is_none(), + "{label} must not contain credentials" + ); + Ok(url) +} + +fn validated_callback_url( + callback_url: &str, + localhost_rewrite: Option<&str>, + allowed_origin: &reqwest::Url, +) -> Result { + let mut callback = parse_http_url(callback_url, "callback URL")?; + if matches!(callback.host_str(), Some("localhost" | "127.0.0.1")) { + if let Some(rewrite) = localhost_rewrite { + callback + .set_host(Some(rewrite)) + .map_err(|_| anyhow::anyhow!("invalid localhost rewrite host"))?; + } + } + anyhow::ensure!( + callback.scheme() == allowed_origin.scheme() + && callback.host_str() == allowed_origin.host_str() + && callback.port_or_known_default() == allowed_origin.port_or_known_default(), + "callback URL origin is not allowed" + ); + anyhow::ensure!( + callback.fragment().is_none(), + "callback URL must not contain a fragment" + ); + Ok(callback) } -async fn call_webhook(callback_url: &str, payload: WebhookPayload) -> Result<()> { +async fn call_webhook( + client: &reqwest::Client, + callback_url: &reqwest::Url, + payload: WebhookPayload, +) -> Result<()> { let e3_id = match &payload { WebhookPayload::Completed { e3_id, .. } => *e3_id, WebhookPayload::Failed { e3_id, .. } => *e3_id, @@ -157,21 +294,17 @@ async fn call_webhook(callback_url: &str, payload: WebhookPayload) -> Result<()> } } - println!("callback_url: {}", callback_url); - - let response = reqwest::Client::new() - .post(callback_url) + let response = client + .post(callback_url.clone()) .json(&payload) .send() .await?; println!("Webhook response status: {}", response.status()); if !response.status().is_success() { - let error_body = response.text().await?; - println!("Webhook error response: {}", error_body); return Err(anyhow::anyhow!( - "Webhook failed with status and body: {}", - error_body + "Webhook failed with status {}", + response.status() )); } @@ -180,9 +313,13 @@ async fn call_webhook(callback_url: &str, payload: WebhookPayload) -> Result<()> Ok(()) } -async fn handle_webhook_delivery(callback_url: &str, payload: WebhookPayload) -> Result<()> { +async fn handle_webhook_delivery( + client: &reqwest::Client, + callback_url: &reqwest::Url, + payload: WebhookPayload, +) -> Result<()> { println!("handle_webhook_delivery()"); - call_webhook(callback_url, payload).await?; + call_webhook(client, callback_url, payload).await?; println!("✓ Webhook sent successfully"); Ok(()) } @@ -190,7 +327,8 @@ async fn handle_webhook_delivery(callback_url: &str, payload: WebhookPayload) -> async fn process_computation_background( runner: Arc, e3_id: u64, - callback_url: &str, + webhook_client: reqwest::Client, + callback_url: reqwest::Url, fhe_inputs: FHEInputs, ) -> Result<()> { match runner(fhe_inputs).await { @@ -202,7 +340,7 @@ async fn process_computation_background( ciphertext, proof, }; - handle_webhook_delivery(callback_url, payload).await?; + handle_webhook_delivery(&webhook_client, &callback_url, payload).await?; println!("✓ Computation completed for E3 {}", e3_id); Ok(()) } @@ -214,7 +352,7 @@ async fn process_computation_background( e3_id, error: format!("Compute failed: {}", error_msg), }; - handle_webhook_delivery(callback_url, payload).await?; + handle_webhook_delivery(&webhook_client, &callback_url, payload).await?; Err(e) } @@ -223,8 +361,10 @@ async fn process_computation_background( async fn handle_compute( config: web::Data, + request: HttpRequest, req: web::Json, ) -> ActixResult { + authorize_compute(&request, &config.bearer_token)?; println!("Processing computation..."); let e3_id = req .e3_id @@ -240,19 +380,22 @@ async fn handle_compute( ciphertexts: req.ciphertext_inputs.clone(), }; - println!("fhe_inputs.params = {:?}", fhe_inputs.params); - let callback_url = if let Some(new_host) = config.localhost_rewrite.clone() { - callback_url - .replace("localhost", &new_host) - .replace("127.0.0.1", &new_host) - } else { - callback_url - }; - println!("callback_url:{}", callback_url); + let callback_url = validated_callback_url( + &callback_url, + config.localhost_rewrite.as_deref(), + &config.callback_origin, + ) + .map_err(actix_web::error::ErrorBadRequest)?; + let permit = Arc::clone(&config.jobs) + .try_acquire_owned() + .map_err(|_| actix_web::error::ErrorTooManyRequests("compute capacity exhausted"))?; let runner = config.runner.clone(); + let webhook_client = config.webhook_client.clone(); tokio::spawn(async move { + let _permit = permit; if let Err(e) = - process_computation_background(runner, e3_id, &callback_url, fhe_inputs).await + process_computation_background(runner, e3_id, webhook_client, callback_url, fhe_inputs) + .await { eprintln!("✗ Background computation failed for E3 {}: {:?}", e3_id, e); } @@ -270,3 +413,83 @@ async fn handle_health_check() -> ActixResult { e3_id: 0, })) } + +#[cfg(test)] +mod server_tests { + use super::*; + use actix_web::test::TestRequest; + + #[test] + fn builder_requires_a_nonempty_bearer_token() { + let missing = E3ProgramServer::builder(|_| async { Ok((vec![], vec![])) }).build(); + assert!(missing.is_err()); + + let empty = E3ProgramServer::builder(|_| async { Ok((vec![], vec![])) }) + .with_bearer_token("") + .build(); + assert!(empty.is_err()); + } + + #[test] + fn callback_validation_rejects_other_origins_and_unsafe_schemes() { + let allowed = reqwest::Url::parse("https://callback.example:8443").unwrap(); + assert!( + validated_callback_url("https://callback.example:8443/results/1", None, &allowed) + .is_ok() + ); + assert!( + validated_callback_url("https://metadata.internal:8443/latest", None, &allowed) + .is_err() + ); + assert!(validated_callback_url("file:///etc/passwd", None, &allowed).is_err()); + } + + #[test] + fn builder_rejects_zero_capacity_and_configures_the_job_limit() { + let zero = E3ProgramServer::builder(|_| async { Ok((vec![], vec![])) }) + .with_bearer_token("secret") + .with_callback_origin("https://callback.example") + .with_max_concurrent_jobs(0) + .build(); + assert!(zero.is_err()); + + let server = E3ProgramServer::builder(|_| async { Ok((vec![], vec![])) }) + .with_bearer_token("secret") + .with_callback_origin("https://callback.example") + .with_max_concurrent_jobs(2) + .build() + .unwrap(); + assert_eq!(server.jobs.available_permits(), 2); + } + + #[test] + fn localhost_rewrite_changes_only_an_exact_local_host() { + let allowed = reqwest::Url::parse("http://host.local:8080").unwrap(); + let rewritten = + validated_callback_url("http://127.0.0.1:8080/result", Some("host.local"), &allowed) + .unwrap(); + assert_eq!(rewritten.as_str(), "http://host.local:8080/result"); + assert!(validated_callback_url( + "http://localhost.attacker:8080/result", + Some("host.local"), + &allowed + ) + .is_err()); + } + + #[test] + fn compute_authorization_accepts_only_the_configured_bearer_token() { + let missing = TestRequest::default().to_http_request(); + assert!(authorize_compute(&missing, "secret").is_err()); + + let wrong = TestRequest::default() + .insert_header((header::AUTHORIZATION, "Bearer wrong")) + .to_http_request(); + assert!(authorize_compute(&wrong, "secret").is_err()); + + let valid = TestRequest::default() + .insert_header((header::AUTHORIZATION, "Bearer secret")) + .to_http_request(); + assert!(authorize_compute(&valid, "secret").is_ok()); + } +} diff --git a/crates/support-scripts/dev/src/main.rs b/crates/support-scripts/dev/src/main.rs index 6a84679cab..66d6a39b37 100644 --- a/crates/support-scripts/dev/src/main.rs +++ b/crates/support-scripts/dev/src/main.rs @@ -4,19 +4,25 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::Result; +use anyhow::{Context, Result}; use e3_program_server::E3ProgramServer; use e3_user_program::fhe_processor; #[tokio::main] async fn main() -> Result<()> { + let bearer_token = std::env::var("INTERFOLD_PROGRAM_SERVER_TOKEN") + .context("INTERFOLD_PROGRAM_SERVER_TOKEN must be set")?; + let callback_origin = std::env::var("INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN") + .context("INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN must be set")?; let server = E3ProgramServer::builder(|inputs| async move { Ok(( vec![3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5], fhe_processor(&inputs), )) }) - .build(); + .with_bearer_token(bearer_token) + .with_callback_origin(callback_origin) + .build()?; server.run().await?; Ok(()) From 94353e5f2195b33d4fb3a6b6007c26df286f8c9c Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 17:26:13 +0500 Subject: [PATCH 36/62] fix(supervisor): report exits and drain children gracefully --- Cargo.lock | 1 + Cargo.toml | 1 + crates/entrypoint/Cargo.toml | 1 + crates/entrypoint/src/fence.rs | 9 + crates/entrypoint/src/nodes/nodes.rs | 38 +++- .../entrypoint/src/nodes/process_manager.rs | 209 ++++++++++++++---- 6 files changed, 214 insertions(+), 45 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d1d8ec1745..39acba8958 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3351,6 +3351,7 @@ dependencies = [ "e3-utils", "e3-zk-prover", "hex", + "libc", "libp2p", "phf", "rand 0.9.2", diff --git a/Cargo.toml b/Cargo.toml index 41814c532b..fb9e59d2df 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -206,6 +206,7 @@ tracing-subscriber = { version = "=0.3.19", features = ["env-filter", "time"] } tracing-test = { version = "=0.2.5", features = ["no-env-filter"] } url = "=2.5.4" vfs = { git = "https://github.com/ryardley/rust-vfs.git", features = ["async-vfs"] } +libc = "=0.2.180" libp2p = { version = "=0.54.1", features = [ "gossipsub", "identify", diff --git a/crates/entrypoint/Cargo.toml b/crates/entrypoint/Cargo.toml index 9f7ff0911c..72f64194e4 100644 --- a/crates/entrypoint/Cargo.toml +++ b/crates/entrypoint/Cargo.toml @@ -27,6 +27,7 @@ hex = { workspace = true } e3-keyshare = { workspace = true } e3-logger = { workspace = true } libp2p = { workspace = true } +libc = { workspace = true } e3-net = { workspace = true } phf = { workspace = true } rand = { workspace = true } diff --git a/crates/entrypoint/src/fence.rs b/crates/entrypoint/src/fence.rs index 3bca1490e2..85a89a939b 100644 --- a/crates/entrypoint/src/fence.rs +++ b/crates/entrypoint/src/fence.rs @@ -43,6 +43,15 @@ pub struct ProcessFence { path: PathBuf, } +impl Drop for ProcessFence { + fn drop(&mut self) { + // Closing normally releases the lock, but an overlapping process spawn can + // briefly inherit the open file description between fork and exec. Unlock + // it explicitly so shutdown makes the fence immediately reacquirable. + let _ = self._file.unlock(); + } +} + impl ProcessFence { /// The path to the lock file being held. pub fn path(&self) -> &Path { diff --git a/crates/entrypoint/src/nodes/nodes.rs b/crates/entrypoint/src/nodes/nodes.rs index 6884cdff1a..bfe095ba19 100644 --- a/crates/entrypoint/src/nodes/nodes.rs +++ b/crates/entrypoint/src/nodes/nodes.rs @@ -31,6 +31,9 @@ pub async fn spawn_process(program: &str, args: Vec) -> Result { .stdin(Stdio::null()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) + // A termination-path error must not turn dropping our last handle into an orphaned + // ciphernode. Normal stops still use SIGTERM and the graceful drain in ProcessManager. + .kill_on_drop(true) .spawn()?; Ok(child) @@ -55,14 +58,47 @@ pub enum Query { Status { status: SwarmStatus }, } -#[derive(Serialize, Deserialize, Clone, Debug)] +#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)] #[serde(rename_all = "lowercase")] pub enum ProcessStatus { Started, Stopped, + Exited { code: Option }, + Unknown, } #[derive(Serialize, Deserialize, Clone, Debug)] pub struct SwarmStatus { pub processes: HashMap, } + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + fn process_exists(pid: u32) -> bool { + // SAFETY: signal 0 performs an existence/permission check and does not mutate the target. + let result = unsafe { libc::kill(pid as libc::pid_t, 0) }; + result == 0 || std::io::Error::last_os_error().raw_os_error() == Some(libc::EPERM) + } + + #[tokio::test] + async fn dropping_the_last_child_handle_does_not_orphan_the_process() { + let child = spawn_process("sh", vec!["-c".into(), "exec sleep 30".into()]) + .await + .unwrap(); + let pid = child.id().unwrap(); + assert!(process_exists(pid)); + + drop(child); + + tokio::time::timeout(Duration::from_secs(5), async { + while process_exists(pid) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("kill-on-drop child should exit promptly"); + } +} diff --git a/crates/entrypoint/src/nodes/process_manager.rs b/crates/entrypoint/src/nodes/process_manager.rs index 10b3e0093c..3765970d5a 100644 --- a/crates/entrypoint/src/nodes/process_manager.rs +++ b/crates/entrypoint/src/nodes/process_manager.rs @@ -4,11 +4,12 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::*; +use anyhow::{bail, Context, Result}; use std::collections::HashMap; use std::io::Write; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use std::time::Duration; use tokio::io::AsyncBufReadExt; use tokio::signal::unix::{signal, SignalKind}; use tokio::sync::Mutex; @@ -23,6 +24,9 @@ use super::nodes::{ spawn_process, CommandMap, ProcessMap, ProcessRecord, ProcessStatus, SwarmStatus, }; +const GRACEFUL_CHILD_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); +const OUTPUT_DRAIN_TIMEOUT: Duration = Duration::from_secs(5); + /// Forward stdout from child process to parent's stdout fn forward_stdout(id: &str, stdout: ChildStdout) -> JoinHandle<()> { let id = id.to_owned(); @@ -89,7 +93,15 @@ async fn run_command(id: &str, program: &str, args: Vec) -> Result Result<()> { let commands = commands.clone(); for (id, (program, args)) in commands { - let record = run_command(&id, &program, args).await?; + let record = match run_command(&id, &program, args).await { + Ok(record) => record, + Err(error) => { + if let Err(cleanup_error) = terminate_processes(processes).await { + error!(%cleanup_error, "Failed to clean up partially started process swarm"); + } + return Err(error).with_context(|| format!("failed to start process {id}")); + } + }; // Store the process let mut processes_guard = processes.lock().await; @@ -100,8 +112,23 @@ async fn run_commands(commands: &CommandMap, processes: &ProcessMap) -> Result<( /// Start a process async fn start(id: &str, commands: &CommandMap, processes: &ProcessMap) -> Result<()> { - if processes.lock().await.contains_key(id) { - bail!("Process {} already running!", id); + { + let mut processes = processes.lock().await; + let exited = if let Some((child, _)) = processes.get_mut(id) { + match child.try_wait().context("failed to inspect child status")? { + None => bail!("Process {} already running!", id), + Some(_) => true, + } + } else { + false + }; + if exited { + if let Some((_, handlers)) = processes.remove(id) { + for handler in handlers { + handler.abort(); + } + } + } } let Some(command) = commands.get(id) else { bail!("Bad command {}", id); @@ -118,53 +145,77 @@ async fn start(id: &str, commands: &CommandMap, processes: &ProcessMap) -> Resul /// Start a process async fn stop(id: &str, processes: &ProcessMap) -> Result<()> { warn!("stopping {}...", id); - let mut processes_lock = processes.lock().await; - if !processes_lock.contains_key(id) { + let process_record = processes.lock().await.remove(id); + let Some(mut process_record) = process_record else { info!("Cannot stop process that isn't running {}", id); return Ok(()); }; - if let Some(mut process_record) = processes_lock.remove(id) { - terminate_process_record(id, &mut process_record).await; - } + terminate_process_record(id, &mut process_record).await?; Ok(()) } -/// Terminate a process -async fn terminate_process_record(id: &str, process_record: &mut ProcessRecord) { +fn send_sigterm(child: &tokio::process::Child) -> Result<()> { + let pid = child.id().context("child has no process id")?; + // SAFETY: `libc::kill` does not dereference pointers. The PID comes from a live + // `tokio::process::Child`, and the signal constant is valid on this Unix-only module. + let result = unsafe { libc::kill(pid as libc::pid_t, libc::SIGTERM) }; + if result == 0 { + return Ok(()); + } + let error = std::io::Error::last_os_error(); + if error.raw_os_error() == Some(libc::ESRCH) { + Ok(()) + } else { + Err(error).context("failed to send SIGTERM to child") + } +} + +/// Ask a process to shut down, then force-kill it only after the grace period. +async fn terminate_process_record(id: &str, process_record: &mut ProcessRecord) -> Result<()> { info!("Terminating {}", id); let (child, handlers) = process_record; - for handler in handlers.drain(..) { - // drop all stdout/in handlers - handler.abort(); - } - if let Err(e) = child.kill().await { - error!("Failed to kill process {}: {}", id, e); + if child.try_wait()?.is_none() { + send_sigterm(child)?; + match tokio::time::timeout(GRACEFUL_CHILD_SHUTDOWN_TIMEOUT, child.wait()).await { + Ok(status) => { + status?; + } + Err(_) => { + warn!( + process = id, + timeout_seconds = GRACEFUL_CHILD_SHUTDOWN_TIMEOUT.as_secs(), + "Child did not exit after SIGTERM; sending SIGKILL" + ); + child.kill().await?; + } + } } - info!("Terminating process: {}...", id); - let _ = child.wait().await; + for mut handler in handlers.drain(..) { + if tokio::time::timeout(OUTPUT_DRAIN_TIMEOUT, &mut handler) + .await + .is_err() + { + handler.abort(); + } + } info!("Process {} terminated.", id); + Ok(()) } /// Terminate all processes -async fn terminate_processes(processes: &ProcessMap) { +async fn terminate_processes(processes: &ProcessMap) -> Result<()> { info!("starting to terminate processes..."); - let keys: Vec = { - processes - .lock() - .await - .keys() - .map(|k| k.to_string()) - .collect() - }; - - let mut processes_guard = processes.lock().await; - for id in keys { - if let Some(mut process_record) = processes_guard.remove(&id) { - terminate_process_record(&id, &mut process_record).await; + let records = std::mem::take(&mut *processes.lock().await); + let mut first_error = None; + for (id, mut process_record) in records { + if let Err(error) = terminate_process_record(&id, &mut process_record).await { + error!(process = id, %error, "Failed to terminate child process"); + first_error.get_or_insert(error); } } + first_error.map_or(Ok(()), Err) } /// Terminate all child processes @@ -172,10 +223,18 @@ async fn terminate_processes_and_exit(processes: &ProcessMap) { let processes = processes.clone(); // taking this off the hot path so we can send a response to the client tokio::spawn(async move { - terminate_processes(&processes).await; - info!("SWARM All processes terminated, exiting"); + let exit_code = match terminate_processes(&processes).await { + Ok(()) => { + info!("SWARM All processes terminated, exiting"); + 0 + } + Err(error) => { + error!(%error, "SWARM child shutdown failed"); + 1 + } + }; let _ = std::io::stdout().flush(); - std::process::exit(0); + std::process::exit(exit_code); }); } @@ -228,8 +287,7 @@ impl ProcessManager { } pub async fn stop_all(&self) -> Result<()> { - terminate_processes(&self.processes).await; - Ok(()) + terminate_processes(&self.processes).await } pub async fn terminate(&self) { @@ -237,11 +295,19 @@ impl ProcessManager { } pub async fn status(&self, id: &str) -> ProcessStatus { - let processes = self.processes.lock().await; - if processes.contains_key(id) { - ProcessStatus::Started - } else { - ProcessStatus::Stopped + let mut processes = self.processes.lock().await; + let Some((child, _)) = processes.get_mut(id) else { + return ProcessStatus::Stopped; + }; + match child.try_wait() { + Ok(None) => ProcessStatus::Started, + Ok(Some(status)) => ProcessStatus::Exited { + code: status.code(), + }, + Err(error) => { + warn!(process = id, %error, "Failed to inspect child process status"); + ProcessStatus::Unknown + } } } @@ -258,7 +324,6 @@ impl ProcessManager { impl From for ProcessManager { fn from(value: CommandMap) -> Self { - // TODO: should probably implement a singleton pattern here but rn it doesn't matter let processes = Arc::new(Mutex::new(HashMap::new())); let manager = Self { commands: value, @@ -270,3 +335,59 @@ impl From for ProcessManager { manager } } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn exited_child_is_not_reported_as_started_and_can_be_started_again() { + let commands = CommandMap::from([( + "short".to_string(), + ( + "sh".to_string(), + vec!["-c".to_string(), "exit 7".to_string()], + ), + )]); + let manager = ProcessManager::from(commands); + manager.start("short").await.unwrap(); + + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + manager.status("short").await, + ProcessStatus::Exited { code: Some(7) } + ); + + manager.start("short").await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert_eq!( + manager.status("short").await, + ProcessStatus::Exited { code: Some(7) } + ); + } + + #[tokio::test] + async fn stop_sends_sigterm_before_forcing_termination() { + let directory = tempfile::tempdir().unwrap(); + let marker = directory.path().join("terminated"); + let script = format!( + "trap 'echo terminated > {} ; exit 0' TERM; while true; do sleep 0.05; done", + marker.display() + ); + let commands = CommandMap::from([( + "long".to_string(), + ("sh".to_string(), vec!["-c".to_string(), script]), + )]); + let manager = ProcessManager::from(commands); + manager.start("long").await.unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + + manager.stop("long").await.unwrap(); + + assert_eq!( + std::fs::read_to_string(marker).unwrap().trim(), + "terminated" + ); + assert_eq!(manager.status("long").await, ProcessStatus::Stopped); + } +} From f047287e7de7b172c82568dcee8a9cac8f75e47d Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 17:40:23 +0500 Subject: [PATCH 37/62] fix(request): preserve and release buffered event order --- crates/request/src/actors/router.rs | 1 + crates/request/src/context.rs | 75 +++++++++++++++++++++-- crates/request/src/domain/event_buffer.rs | 62 +++++++++++++------ 3 files changed, 115 insertions(+), 23 deletions(-) diff --git a/crates/request/src/actors/router.rs b/crates/request/src/actors/router.rs index 5f1a1f3a8a..6246caa44d 100644 --- a/crates/request/src/actors/router.rs +++ b/crates/request/src/actors/router.rs @@ -188,6 +188,7 @@ impl Handler for E3Router { PostForward::Teardown => { // The original event is forwarded above so children can kill themselves. self.contexts.remove(&e3_id); + self.buffer.remove_e3(&e3_id); self.completed.insert(e3_id); } PostForward::None => (), diff --git a/crates/request/src/context.rs b/crates/request/src/context.rs index bd2c6d155f..426bd643c5 100644 --- a/crates/request/src/context.rs +++ b/crates/request/src/context.rs @@ -82,15 +82,15 @@ impl E3Context { pub fn forward_message(&self, msg: &InterfoldEvent, buffer: &mut EventBuffer) { self.recipients().into_iter().for_each(|(key, recipient)| { - // Use composite key of e3_id:recipient_key to scope buffered events per E3 request - let buffer_key = format!("{}:{}", self.e3_id, key); if let Some(act) = recipient { - act.do_send(msg.clone()); - for m in buffer.take(&buffer_key) { + // Events buffered before this extension existed must remain ahead of the event + // that caused the extension to be constructed. + for m in buffer.take(&self.e3_id, &key) { act.do_send(m); } + act.do_send(msg.clone()); } else { - buffer.add(&buffer_key, msg.clone()); + buffer.add(&self.e3_id, &key, msg.clone()); } }); } @@ -180,3 +180,68 @@ impl Checkpoint for E3Context { &self.repository } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::ContextRepositoryFactory; + use actix::{Actor, Context, Handler, Message}; + use e3_data::{DataStore, InMemStore}; + use e3_events::{Event, InterfoldEventData, Sequenced}; + use std::sync::{Arc, Mutex}; + + struct Recorder(Arc>>); + + impl Actor for Recorder { + type Context = Context; + } + + impl Handler for Recorder { + type Result = (); + + fn handle(&mut self, message: InterfoldEvent, _: &mut Self::Context) { + if let InterfoldEventData::TestEvent(event) = message.get_data() { + self.0.lock().unwrap().push(event.msg.clone()); + } + } + } + + #[derive(Message)] + #[rtype(result = "Vec")] + struct Recorded; + + impl Handler for Recorder { + type Result = Vec; + + fn handle(&mut self, _: Recorded, _: &mut Self::Context) -> Self::Result { + self.0.lock().unwrap().clone() + } + } + + fn event(e3_id: &E3id, label: &str, sequence: u64) -> InterfoldEvent { + InterfoldEvent::::test_event(label) + .e3_id(e3_id.clone()) + .seq(sequence) + .build() + } + + #[actix::test] + async fn newly_attached_recipient_observes_buffered_events_before_current_event() { + let e3_id = E3id::new("7", 1); + let store = DataStore::from_in_mem(&InMemStore::new(false).start()); + let mut context = E3Context::from_params(E3ContextParams { + repository: store.repositories().context(&e3_id), + e3_id: e3_id.clone(), + extensions: Arc::new(Vec::new()), + }); + let recorded = Arc::new(Mutex::new(Vec::new())); + let recorder = Recorder(recorded).start(); + let mut buffer = EventBuffer::default(); + buffer.add(&e3_id, "keyshare", event(&e3_id, "older", 1)); + context.set_event_recipient("keyshare", Some(recorder.clone().recipient())); + + context.forward_message(&event(&e3_id, "current", 2), &mut buffer); + + assert_eq!(recorder.send(Recorded).await.unwrap(), ["older", "current"]); + } +} diff --git a/crates/request/src/domain/event_buffer.rs b/crates/request/src/domain/event_buffer.rs index b966bce7b1..a7d3fb0fe9 100644 --- a/crates/request/src/domain/event_buffer.rs +++ b/crates/request/src/domain/event_buffer.rs @@ -4,28 +4,36 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use e3_events::InterfoldEvent; +use e3_events::{E3id, InterfoldEvent}; use std::collections::HashMap; /// Buffers events for downstream instances to handle out-of-order event delivery. -/// Events are stored in a HashMap keyed by string identifiers until they are ready -/// to be processed. +/// Events are scoped by protocol request and recipient until the recipient is ready. #[derive(Default)] pub struct EventBuffer { - buffer: HashMap>, + buffer: HashMap<(E3id, String), Vec>, } impl EventBuffer { - pub fn add(&mut self, key: &str, event: InterfoldEvent) { - self.buffer.entry(key.to_string()).or_default().push(event) + pub fn add(&mut self, e3_id: &E3id, recipient: &str, event: InterfoldEvent) { + self.buffer + .entry((e3_id.clone(), recipient.to_owned())) + .or_default() + .push(event) } - pub fn take(&mut self, key: &str) -> Vec { + pub fn take(&mut self, e3_id: &E3id, recipient: &str) -> Vec { self.buffer - .get_mut(key) - .map(std::mem::take) + .remove(&(e3_id.clone(), recipient.to_owned())) .unwrap_or_default() } + + /// Discard data for a terminal request, including recipients that were never constructed on + /// this node role. + pub fn remove_e3(&mut self, e3_id: &E3id) { + self.buffer + .retain(|(buffered_id, _), _| buffered_id != e3_id); + } } #[cfg(test)] @@ -43,28 +51,46 @@ mod tests { #[test] fn take_returns_empty_for_unknown_key() { let mut buffer = EventBuffer::default(); - assert!(buffer.take("missing").is_empty()); + assert!(buffer.take(&E3id::new("1", 1), "missing").is_empty()); } #[test] fn add_then_take_drains_buffer() { let mut buffer = EventBuffer::default(); - buffer.add("k", event("a")); - buffer.add("k", event("b")); + let e3_id = E3id::new("1", 1); + buffer.add(&e3_id, "k", event("a")); + buffer.add(&e3_id, "k", event("b")); - let drained = buffer.take("k"); + let drained = buffer.take(&e3_id, "k"); assert_eq!(drained.len(), 2); // A second take should yield nothing since the buffer was drained. - assert!(buffer.take("k").is_empty()); + assert!(buffer.take(&e3_id, "k").is_empty()); } #[test] fn keys_are_isolated() { let mut buffer = EventBuffer::default(); - buffer.add("a", event("x")); - buffer.add("b", event("y")); + let e3_id = E3id::new("1", 1); + buffer.add(&e3_id, "a", event("x")); + buffer.add(&e3_id, "b", event("y")); + + assert_eq!(buffer.take(&e3_id, "a").len(), 1); + assert_eq!(buffer.take(&e3_id, "b").len(), 1); + } + + #[test] + fn terminal_cleanup_removes_only_the_completed_e3() { + let mut buffer = EventBuffer::default(); + let completed = E3id::new("1", 1); + let active = E3id::new("2", 1); + buffer.add(&completed, "missing-a", event("old-a")); + buffer.add(&completed, "missing-b", event("old-b")); + buffer.add(&active, "missing-a", event("active")); + + buffer.remove_e3(&completed); - assert_eq!(buffer.take("a").len(), 1); - assert_eq!(buffer.take("b").len(), 1); + assert!(buffer.take(&completed, "missing-a").is_empty()); + assert!(buffer.take(&completed, "missing-b").is_empty()); + assert_eq!(buffer.take(&active, "missing-a").len(), 1); } } From 165774580a2e328b21eb1006dcff032991d4dd25 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 17:47:18 +0500 Subject: [PATCH 38/62] fix(sync): fence startup recovery phases --- crates/sync/src/actors/sync.rs | 61 ++++++++++++++++++++++++++++------ 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/crates/sync/src/actors/sync.rs b/crates/sync/src/actors/sync.rs index c20f675594..8d8bd1a75a 100644 --- a/crates/sync/src/actors/sync.rs +++ b/crates/sync/src/actors/sync.rs @@ -156,23 +156,35 @@ pub async fn sync( SyncPlanner::sort_by_timestamp(&mut historical); info!("Historical events sorted."); - // 8. Enable effects + // 8-10. Enable effects, publish canonical history, then enter live mode. Each phase is fenced + // through durable storage and EventBus fanout so aggregate-specific EventStore response order + // cannot move history ahead of EffectsEnabled or SyncEnded ahead of history. + publish_reconciled_history(bus, historical).await?; + // normal live operations + + Ok(()) +} + +async fn publish_reconciled_history( + bus: &BusHandle, + historical: Vec>, +) -> Result<()> { + info!("Enabling effects..."); bus.publish_without_context(EffectsEnabled::new())?; - info!("Effects enabled"); + bus.flush_event_pipeline().await?; + info!("Effects enabled."); - // 9. Publish the new sorted events to the eventstore info!("Publishing historical events to actors..."); for event in historical { bus.naked_dispatch_async(event).await?; } + bus.flush_event_pipeline().await?; info!("Historical events published."); - // 10. Publish the SyncEnded event info!("Publishing SyncEnded event..."); bus.publish_without_context(SyncEnded::new())?; + bus.flush_event_pipeline().await?; info!("Sync finished."); - // normal live operations - Ok(()) } @@ -304,8 +316,8 @@ impl SnapshotLoaded { #[cfg(test)] mod tests { use super::{ - collect_historical_evm_events, preflight_schema_version, seed_clock_from_replay, - sort_replay_events, + collect_historical_evm_events, preflight_schema_version, publish_reconciled_history, + seed_clock_from_replay, sort_replay_events, }; use crate::domain::SyncPlanner; use e3_ciphernode_builder::EventSystem; @@ -313,8 +325,8 @@ mod tests { use e3_events::{ hlc::{Hlc, HlcTimestamp}, EffectsEnabled, Event, EventPublisher, EventSubscriber, EventType, EvmEventConfig, - EvmEventConfigChain, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, InterfoldEvent, - InterfoldEventData, SyncEnded, TakeEvents, Unsequenced, + EvmEventConfigChain, GetEvents, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, + InterfoldEvent, InterfoldEventData, SyncEnded, TakeEvents, Unsequenced, }; use std::collections::BTreeMap; @@ -537,6 +549,35 @@ mod tests { assert_eq!(timestamps, vec![10, 20, 30]); Ok(()) } + #[actix::test] + async fn startup_history_is_fenced_between_effects_and_live_mode() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-startup-history-fences"); + let history = bus.history(); + let historical = vec![ + InterfoldEvent::::test_event("first") + .id(1) + .ts(10) + .build(), + InterfoldEvent::::test_event("second") + .id(2) + .ts(20) + .build(), + ]; + + publish_reconciled_history(&bus, historical).await?; + + let received = history.send(GetEvents::new()).await?; + let types = received + .iter() + .map(|event| event.event_type()) + .collect::>(); + assert_eq!( + types, + ["EffectsEnabled", "TestEvent", "TestEvent", "SyncEnded"] + ); + Ok(()) + } /// Verify that `run_once::` correctly gates event subscriptions. /// From 94bc1804902d4bb6b27caa83d464d3fe478051dc Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 18:25:13 +0500 Subject: [PATCH 39/62] fix(zk): count C3 proofs by threshold share rows --- .../src/domain/share_verification.rs | 96 ++++++++++++++----- 1 file changed, 71 insertions(+), 25 deletions(-) diff --git a/crates/zk-prover/src/domain/share_verification.rs b/crates/zk-prover/src/domain/share_verification.rs index f9b50f5fc0..9e3c8dc1eb 100644 --- a/crates/zk-prover/src/domain/share_verification.rs +++ b/crates/zk-prover/src/domain/share_verification.rs @@ -196,10 +196,10 @@ impl ShareVerifier { /// Check that a party supplied the canonical proof-type layout for this protocol phase. /// - /// C2/C3 counts are derived from the DKG parameter preset. Variable C4b and C6 counts are - /// checked against trusted local state by their producers. This trust-boundary check prevents - /// a signed proof for another phase (or a duplicated singleton proof) from satisfying the - /// current phase merely because its self-declared [`ProofType`] maps to a valid circuit. + /// C2/C3 counts are derived from the threshold parameter preset. Variable C4b and C6 counts + /// are checked against trusted local state by their producers. This trust-boundary check + /// prevents a signed proof for another phase (or a duplicated singleton proof) from satisfying + /// the current phase merely because its self-declared [`ProofType`] maps to a valid circuit. fn has_canonical_proof_shape( kind: &VerificationKind, signed_proofs: &[SignedProofPayload], @@ -211,17 +211,21 @@ impl ShareVerifier { && signed_proofs[0].payload.proof_type == ProofType::C1PkGeneration } VerificationKind::ShareProofs => { - // Canonical order is C2a, C2b, C3a x L, C3b x L. Share encryption uses the - // DKG counterpart of a threshold preset (or the preset itself when already DKG). - let dkg_preset = params_preset.dkg_counterpart().unwrap_or(params_preset); - let num_moduli = dkg_preset.metadata().num_moduli; - signed_proofs.len() == 2 + (2 * num_moduli) + // Canonical order is C2a, C2b, C3a x L, C3b x L. Each C3 proof encrypts one + // modulus row of the threshold-parameter Shamir secret, even though encryption + // itself uses the paired DKG parameters. The dispatch currently carries that DKG + // preset, so recover its threshold counterpart before deriving L. + let threshold_preset = params_preset + .threshold_counterpart() + .unwrap_or(params_preset); + let num_share_rows = threshold_preset.metadata().num_moduli; + signed_proofs.len() == 2 + (2 * num_share_rows) && signed_proofs[0].payload.proof_type == ProofType::C2aSkShareComputation && signed_proofs[1].payload.proof_type == ProofType::C2bESmShareComputation - && signed_proofs[2..2 + num_moduli] + && signed_proofs[2..2 + num_share_rows] .iter() .all(|signed| signed.payload.proof_type == ProofType::C3aSkShareEncryption) - && signed_proofs[2 + num_moduli..] + && signed_proofs[2 + num_share_rows..] .iter() .all(|signed| signed.payload.proof_type == ProofType::C3bESmShareEncryption) } @@ -693,6 +697,24 @@ mod tests { SignedProofPayload::sign(payload, s).expect("sign") } + fn signed_share_bundle( + s: &PrivateKeySigner, + e3_id: &E3id, + num_share_rows: usize, + ) -> Vec { + let mut proofs = vec![ + signed_proof(s, e3_id, ProofType::C2aSkShareComputation, 2), + signed_proof(s, e3_id, ProofType::C2bESmShareComputation, 3), + ]; + for _ in 0..num_share_rows { + proofs.push(signed_proof(s, e3_id, ProofType::C3aSkShareEncryption, 4)); + } + for _ in 0..num_share_rows { + proofs.push(signed_proof(s, e3_id, ProofType::C3bESmShareEncryption, 5)); + } + proofs + } + /// Build a signed C1 (PkGeneration) proof for `party_id` under `e3_id`, /// optionally with a deliberately wrong circuit name. fn signed_pk(s: &PrivateKeySigner, e3_id: &E3id, wrong_circuit: bool) -> SignedProofPayload { @@ -893,12 +915,7 @@ mod tests { BfvPreset::InsecureDkg512, )); - let share_bundle = vec![ - signed_proof(&s, &e3, ProofType::C2aSkShareComputation, 2), - signed_proof(&s, &e3, ProofType::C2bESmShareComputation, 3), - signed_proof(&s, &e3, ProofType::C3aSkShareEncryption, 4), - signed_proof(&s, &e3, ProofType::C3bESmShareEncryption, 5), - ]; + let share_bundle = signed_share_bundle(&s, &e3, 2); assert!(ShareVerifier::has_canonical_proof_shape( &VerificationKind::ShareProofs, &share_bundle, @@ -912,14 +929,7 @@ mod tests { BfvPreset::InsecureDkg512, )); - let secure_share_bundle = vec![ - signed_proof(&s, &e3, ProofType::C2aSkShareComputation, 10), - signed_proof(&s, &e3, ProofType::C2bESmShareComputation, 11), - signed_proof(&s, &e3, ProofType::C3aSkShareEncryption, 12), - signed_proof(&s, &e3, ProofType::C3aSkShareEncryption, 13), - signed_proof(&s, &e3, ProofType::C3bESmShareEncryption, 14), - signed_proof(&s, &e3, ProofType::C3bESmShareEncryption, 15), - ]; + let secure_share_bundle = signed_share_bundle(&s, &e3, 3); assert!(ShareVerifier::has_canonical_proof_shape( &VerificationKind::ShareProofs, &secure_share_bundle, @@ -961,6 +971,42 @@ mod tests { )); } + #[test] + fn share_shape_uses_threshold_secret_rows_when_dispatch_carries_dkg_preset() { + let s = signer(); + let e3 = e3(); + + // Production dispatch carries the share-encryption (DKG) preset, but C3 requests are + // generated from rows of the paired threshold-parameter Shamir secret. + let insecure_production_bundle = signed_share_bundle(&s, &e3, 2); + assert_eq!(BfvPreset::InsecureDkg512.metadata().num_moduli, 1); + assert_eq!(BfvPreset::InsecureThreshold512.metadata().num_moduli, 2); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &insecure_production_bundle, + BfvPreset::InsecureDkg512, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &signed_share_bundle(&s, &e3, 1), + BfvPreset::InsecureDkg512, + )); + + let secure_production_bundle = signed_share_bundle(&s, &e3, 3); + assert_eq!(BfvPreset::SecureDkg8192.metadata().num_moduli, 2); + assert_eq!(BfvPreset::SecureThreshold8192.metadata().num_moduli, 3); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &secure_production_bundle, + BfvPreset::SecureDkg8192, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &signed_share_bundle(&s, &e3, 2), + BfvPreset::SecureDkg8192, + )); + } + #[test] fn prepare_excludes_wrong_phase_without_creating_slash_evidence() { let s = signer(); From 3c95c64726ea53ce6fcc5129f6ed833e3586a082 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 18:55:25 +0500 Subject: [PATCH 40/62] test(zk): use committee-scoped node-fold artifacts --- crates/zk-prover/tests/node_fold_correlated_e2e_tests.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/zk-prover/tests/node_fold_correlated_e2e_tests.rs b/crates/zk-prover/tests/node_fold_correlated_e2e_tests.rs index 6f70507106..45d3c49f36 100644 --- a/crates/zk-prover/tests/node_fold_correlated_e2e_tests.rs +++ b/crates/zk-prover/tests/node_fold_correlated_e2e_tests.rs @@ -157,7 +157,8 @@ async fn node_fold_correlated_sparse_self_slot_proves_and_verifies() { let (backend, temp) = setup_test_prover(&bb).await; let prover = ZkProver::new(&backend); - let artifacts_dir = preset.artifacts_dir(); + let artifacts_dir = + preset.artifacts_dir_for_committee(CiphernodesCommitteeSize::Minimum.as_str()); for g in [ "pk", From 1d2ecd0d5c6b5461e510dc84cc1c40201f68aaae Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 19:20:46 +0500 Subject: [PATCH 41/62] fix(deps): apply compatible RustSec patches --- Cargo.lock | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39acba8958..9263ea8a31 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2207,9 +2207,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" +checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33" dependencies = [ "serde", ] @@ -2680,9 +2680,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.18" +version = "0.9.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" dependencies = [ "crossbeam-utils", ] @@ -6797,9 +6797,9 @@ dependencies = [ [[package]] name = "num-conv" -version = "0.1.0" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" [[package]] name = "num-integer" @@ -7900,9 +7900,9 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.13" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", "getrandom 0.3.4", @@ -8518,7 +8518,7 @@ dependencies = [ "once_cell", "ring 0.17.14", "rustls-pki-types", - "rustls-webpki 0.103.9", + "rustls-webpki 0.103.13", "subtle", "zeroize", ] @@ -8545,9 +8545,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.9" +version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ "ring 0.17.14", "rustls-pki-types", @@ -9367,9 +9367,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tar" -version = "0.4.44" +version = "0.4.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d863878d212c87a19c1a610eb53bb01fe12951c0501cf5a0d65f724914a667a" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" dependencies = [ "filetime", "libc", @@ -9469,9 +9469,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.45" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", @@ -9484,15 +9484,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" +checksum = "7694e1cfe791f8d31026952abf09c69ca6f6fa4e1a1229e18988f06a04a12dca" [[package]] name = "time-macros" -version = "0.2.25" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +checksum = "2e70e4c5a0e0a8a4823ad65dfe1a6930e4f4d756dcd9dd7939022b5e8c501215" dependencies = [ "num-conv", "time-core", From 3e98e08f2018dfa6fd41fd4e1a9c19cf51c50821 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 19:22:21 +0500 Subject: [PATCH 42/62] fix(logging): update tracing subscriber --- Cargo.lock | 66 ++++------ Cargo.toml | 2 +- crates/cli/src/helpers/telemetry.rs | 196 +++++++++++++++++++++++++++- crates/support/Cargo.toml | 2 +- 4 files changed, 218 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9263ea8a31..be01d98f7a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1235,7 +1235,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1246,7 +1246,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2831,7 +2831,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.116", ] [[package]] @@ -2994,7 +2994,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -4209,7 +4209,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5523,7 +5523,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6279,11 +6279,11 @@ dependencies = [ [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] @@ -6735,12 +6735,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "overload", - "winapi", + "windows-sys 0.61.2", ] [[package]] @@ -7083,15 +7082,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "owo-colors" version = "4.2.3" @@ -7653,7 +7646,7 @@ dependencies = [ "rand 0.9.2", "rand_chacha 0.9.0", "rand_xorshift 0.4.0", - "regex-syntax 0.8.8", + "regex-syntax", "rusty-fork", "tempfile", "unarray", @@ -8195,17 +8188,8 @@ checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.13", - "regex-syntax 0.8.8", -] - -[[package]] -name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", + "regex-automata", + "regex-syntax", ] [[package]] @@ -8216,7 +8200,7 @@ checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.8", + "regex-syntax", ] [[package]] @@ -8225,12 +8209,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" -[[package]] -name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - [[package]] name = "regex-syntax" version = "0.8.8" @@ -8506,7 +8484,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9895,14 +9873,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "sharded-slab", "smallvec", "thread_local", @@ -10367,7 +10345,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fb9e59d2df..d1935ba1bb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -202,7 +202,7 @@ tokio = { version = "=1.46.1", features = ["full"] } toml = "=0.8.23" tracing = "=0.1.41" tracing-opentelemetry = "=0.30.0" -tracing-subscriber = { version = "=0.3.19", features = ["env-filter", "time"] } +tracing-subscriber = { version = "=0.3.20", features = ["env-filter", "time"] } tracing-test = { version = "=0.2.5", features = ["no-env-filter"] } url = "=2.5.4" vfs = { git = "https://github.com/ryardley/rust-vfs.git", features = ["async-vfs"] } diff --git a/crates/cli/src/helpers/telemetry.rs b/crates/cli/src/helpers/telemetry.rs index 6721ec6be5..cff9fe3023 100644 --- a/crates/cli/src/helpers/telemetry.rs +++ b/crates/cli/src/helpers/telemetry.rs @@ -10,9 +10,14 @@ use e3_logger::{LogCollector, OperationalLogLayer}; use opentelemetry::trace::TracerProvider; use opentelemetry_otlp::{Protocol, WithExportConfig}; use opentelemetry_sdk::{trace::SdkTracerProvider, Resource}; -use std::path::PathBuf; -use tracing::Level; +use std::{fmt, path::PathBuf}; +use tracing::{ + field::{Field, Visit}, + Level, +}; +use tracing_subscriber::field::RecordFields; use tracing_subscriber::filter::Targets; +use tracing_subscriber::fmt::format::{FormatFields, Writer}; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; pub fn setup_simple_tracing(log_level: Level) { @@ -24,6 +29,8 @@ pub fn setup_simple_tracing(log_level: Level) { .with( tracing_subscriber::fmt::layer() .compact() + .with_ansi(true) + .fmt_fields(TerminalFields) .with_target(false), ) .with(OperationalLogLayer) @@ -57,6 +64,8 @@ pub fn setup_tracing(config: &AppConfig, log_level: Level) -> Result<()> { .with( tracing_subscriber::fmt::layer() .compact() + .with_ansi(true) + .fmt_fields(TerminalFields) .with_target(false), ) .with(OperationalLogLayer) @@ -69,6 +78,8 @@ pub fn setup_tracing(config: &AppConfig, log_level: Level) -> Result<()> { .with( tracing_subscriber::fmt::layer() .compact() + .with_ansi(true) + .fmt_fields(TerminalFields) .with_target(false), ) .with(OperationalLogLayer) @@ -87,3 +98,184 @@ fn operational_log_path(config: &AppConfig) -> PathBuf { .unwrap_or_else(|| PathBuf::from(".")) .join("ciphernode.jsonl") } + +#[derive(Clone, Copy, Debug, Default)] +struct TerminalFields; + +impl<'writer> FormatFields<'writer> for TerminalFields { + fn format_fields(&self, writer: Writer<'writer>, fields: R) -> fmt::Result { + let mut visitor = TerminalFieldVisitor::new(writer); + fields.record(&mut visitor); + visitor.finish() + } +} + +struct TerminalFieldVisitor<'writer> { + writer: Writer<'writer>, + is_empty: bool, + result: fmt::Result, +} + +impl<'writer> TerminalFieldVisitor<'writer> { + fn new(writer: Writer<'writer>) -> Self { + Self { + writer, + is_empty: true, + result: Ok(()), + } + } + + fn finish(self) -> fmt::Result { + self.result + } + + fn begin_field(&mut self) -> bool { + if self.result.is_err() { + return false; + } + + if self.is_empty { + self.is_empty = false; + true + } else { + self.result = write!(self.writer, " "); + self.result.is_ok() + } + } + + fn record_message_debug(&mut self, value: &dyn fmt::Debug) { + if self.begin_field() { + self.result = write!(self.writer, "{value:?}"); + } + } + + fn record_message_display(&mut self, value: impl fmt::Display) { + if self.begin_field() { + self.result = write!(self.writer, "{value}"); + } + } + + fn record_named_debug(&mut self, name: &str, value: &dyn fmt::Debug) { + if !self.begin_field() { + return; + } + + let name = name.strip_prefix("r#").unwrap_or(name); + self.result = write!(self.writer, "{name}={value:?}"); + } + + fn record_named_display(&mut self, name: &str, value: impl fmt::Display) { + if !self.begin_field() { + return; + } + + let name = name.strip_prefix("r#").unwrap_or(name); + self.result = write!(self.writer, "{name}={value}"); + } +} + +impl Visit for TerminalFieldVisitor<'_> { + fn record_str(&mut self, field: &Field, value: &str) { + if field.name() == "message" { + self.record_message_display(value); + } else { + self.record_named_debug(field.name(), &value); + } + } + + fn record_bytes(&mut self, field: &Field, value: &[u8]) { + self.record_named_debug(field.name(), &value); + } + + fn record_error(&mut self, field: &Field, value: &(dyn std::error::Error + 'static)) { + if field.name() == "message" { + self.record_message_display(value); + } else { + self.record_named_display(field.name(), value); + } + } + + fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) { + let name = field.name(); + + if name.starts_with("log.") { + return; + } + + if name == "message" { + self.record_message_debug(value); + } else { + self.record_named_debug(name, value); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{ + io, + sync::{Arc, Mutex}, + }; + use tracing_subscriber::{fmt::writer::MakeWriter, layer::SubscriberExt}; + + #[derive(Clone, Default)] + struct CapturedLogs { + bytes: Arc>>, + } + + impl CapturedLogs { + fn contents(&self) -> String { + let bytes = self.bytes.lock().expect("captured logs lock poisoned"); + String::from_utf8(bytes.clone()).expect("captured logs should be valid UTF-8") + } + } + + struct CapturedLogWriter { + bytes: Arc>>, + } + + impl io::Write for CapturedLogWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.bytes + .lock() + .expect("captured logs lock poisoned") + .extend_from_slice(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + impl<'writer> MakeWriter<'writer> for CapturedLogs { + type Writer = CapturedLogWriter; + + fn make_writer(&'writer self) -> Self::Writer { + CapturedLogWriter { + bytes: Arc::clone(&self.bytes), + } + } + } + + #[test] + fn terminal_fields_preserve_ansi_escape_bytes_in_messages() { + let captured = CapturedLogs::default(); + let subscriber = tracing_subscriber::registry().with( + tracing_subscriber::fmt::layer() + .compact() + .with_ansi(false) + .fmt_fields(TerminalFields) + .with_writer(captured.clone()), + ); + + tracing::subscriber::with_default(subscriber, || { + tracing::info!("{}\x1b[33mcolored\x1b[0m", "terminal "); + }); + + let output = captured.contents(); + assert!(output.contains("\x1b[33mcolored\x1b[0m")); + assert!(!output.contains("\\x1b[33mcolored\\x1b[0m")); + } +} diff --git a/crates/support/Cargo.toml b/crates/support/Cargo.toml index 54769d2db1..721ce1c389 100644 --- a/crates/support/Cargo.toml +++ b/crates/support/Cargo.toml @@ -46,7 +46,7 @@ fhe-util = { git = "https://github.com/gnosisguild/fhe.rs", tag = "v0.2.2" } e3-compute-provider = { git = "https://github.com/theinterfold/interfold", tag = "v0.2.2" } tokio = { version = "=1.46.1", features = ["full"] } rand = { version = "=0.9.2" } -tracing-subscriber = { version = "=0.3.19", features = ["env-filter"] } +tracing-subscriber = { version = "=0.3.20", features = ["env-filter"] } boundless-market = { version = "=1.1.0" } url = { version = "=2.5.4" } dotenvy = { version = "=0.15.7" } From c3facf66153a047d6c652d37051e3bcee28b53fd Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 11:30:52 +0500 Subject: [PATCH 43/62] fix(logging): redact sensitive tracing fields --- crates/cli/Cargo.toml | 3 + crates/cli/src/helpers/telemetry.rs | 82 +++++++++++++++-- crates/logger/src/tracing_layer.rs | 133 ++++++++++++++++++++++++++++ 3 files changed, 212 insertions(+), 6 deletions(-) diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 28e37b9070..7239c9df35 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -58,3 +58,6 @@ zeroize = { workspace = true } [build-dependencies] serde_json = { workspace = true } + +[dev-dependencies] +opentelemetry_sdk = { workspace = true, features = ["testing"] } diff --git a/crates/cli/src/helpers/telemetry.rs b/crates/cli/src/helpers/telemetry.rs index cff9fe3023..0d1c50a0d1 100644 --- a/crates/cli/src/helpers/telemetry.rs +++ b/crates/cli/src/helpers/telemetry.rs @@ -6,7 +6,10 @@ use anyhow::Result; use e3_config::AppConfig; -use e3_logger::{LogCollector, OperationalLogLayer}; +use e3_logger::{ + is_sensitive_field, telemetry_metadata_is_safe, LogCollector, OperationalLogLayer, + REDACTED_FIELD_VALUE, +}; use opentelemetry::trace::TracerProvider; use opentelemetry_otlp::{Protocol, WithExportConfig}; use opentelemetry_sdk::{trace::SdkTracerProvider, Resource}; @@ -18,7 +21,7 @@ use tracing::{ use tracing_subscriber::field::RecordFields; use tracing_subscriber::filter::Targets; use tracing_subscriber::fmt::format::{FormatFields, Writer}; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, Layer}; pub fn setup_simple_tracing(log_level: Level) { LogCollector::init("interfold", None); @@ -57,8 +60,11 @@ pub fn setup_tracing(config: &AppConfig, log_level: Level) -> Result<()> { .with_batch_exporter(exporter) .with_resource(Resource::builder().with_service_name(name).build()) .build(); - let telemetry = - tracing_opentelemetry::layer().with_tracer(provider.tracer("interfold-ciphernode")); + let telemetry = tracing_opentelemetry::layer() + .with_tracer(provider.tracer("interfold-ciphernode")) + .with_filter(tracing_subscriber::filter::filter_fn( + telemetry_metadata_is_safe, + )); let _ = tracing_subscriber::registry() .with( @@ -161,7 +167,11 @@ impl<'writer> TerminalFieldVisitor<'writer> { } let name = name.strip_prefix("r#").unwrap_or(name); - self.result = write!(self.writer, "{name}={value:?}"); + if is_sensitive_field(name) { + self.result = write!(self.writer, "{name}={REDACTED_FIELD_VALUE:?}"); + } else { + self.result = write!(self.writer, "{name}={value:?}"); + } } fn record_named_display(&mut self, name: &str, value: impl fmt::Display) { @@ -170,7 +180,11 @@ impl<'writer> TerminalFieldVisitor<'writer> { } let name = name.strip_prefix("r#").unwrap_or(name); - self.result = write!(self.writer, "{name}={value}"); + if is_sensitive_field(name) { + self.result = write!(self.writer, "{name}={REDACTED_FIELD_VALUE}"); + } else { + self.result = write!(self.writer, "{name}={value}"); + } } } @@ -213,6 +227,7 @@ impl Visit for TerminalFieldVisitor<'_> { #[cfg(test)] mod tests { use super::*; + use opentelemetry_sdk::trace::InMemorySpanExporter; use std::{ io, sync::{Arc, Mutex}, @@ -278,4 +293,59 @@ mod tests { assert!(output.contains("\x1b[33mcolored\x1b[0m")); assert!(!output.contains("\\x1b[33mcolored\\x1b[0m")); } + + #[test] + fn terminal_fields_redact_sensitive_values() { + let captured = CapturedLogs::default(); + let subscriber = tracing_subscriber::registry().with( + tracing_subscriber::fmt::layer() + .compact() + .with_ansi(false) + .fmt_fields(TerminalFields) + .with_writer(captured.clone()), + ); + + tracing::subscriber::with_default(subscriber, || { + tracing::info!( + private_key = "terminal-private-key-sentinel", + tx_hash = "0xsafe" + ); + }); + + let output = captured.contents(); + assert!(!output.contains("terminal-private-key-sentinel")); + assert!(output.contains(REDACTED_FIELD_VALUE)); + assert!(output.contains("0xsafe")); + } + + #[test] + fn telemetry_layer_excludes_sensitive_callsites() { + let exporter = InMemorySpanExporter::default(); + let provider = SdkTracerProvider::builder() + .with_simple_exporter(exporter.clone()) + .build(); + let telemetry = tracing_opentelemetry::layer() + .with_tracer(provider.tracer("redaction-test")) + .with_filter(tracing_subscriber::filter::filter_fn( + telemetry_metadata_is_safe, + )); + let subscriber = tracing_subscriber::registry().with(telemetry); + + tracing::subscriber::with_default(subscriber, || { + let sensitive = tracing::info_span!( + "sensitive-otel-span", + private_key = "otel-private-key-sentinel" + ); + drop(sensitive); + let safe = tracing::info_span!("safe-otel-span", e3_id = "31337:1"); + drop(safe); + }); + provider.force_flush().expect("flush in-memory exporter"); + + let spans = exporter.get_finished_spans().expect("exported spans"); + let rendered = format!("{spans:?}"); + assert!(!rendered.contains("otel-private-key-sentinel")); + assert!(!rendered.contains("sensitive-otel-span")); + assert!(rendered.contains("safe-otel-span")); + } } diff --git a/crates/logger/src/tracing_layer.rs b/crates/logger/src/tracing_layer.rs index 5b6631e984..f0c28ba6e0 100644 --- a/crates/logger/src/tracing_layer.rs +++ b/crates/logger/src/tracing_layer.rs @@ -21,6 +21,59 @@ const MAX_FIELD_VALUE_CHARS: usize = 1024; /// growth from inheriting every ancestor span's fields. const MAX_FIELDS: usize = 64; +pub const REDACTED_FIELD_VALUE: &str = "[REDACTED]"; + +/// Field-name based fail-safe for values that must never reach the in-memory log store or JSONL +/// exporter. Tracing field names are static callsite metadata, so this check is deterministic and +/// runs before values are retained or inherited by child spans. +pub fn is_sensitive_field(name: &str) -> bool { + let normalized: String = name + .chars() + .filter(|ch| ch.is_ascii_alphanumeric()) + .flat_map(char::to_lowercase) + .collect(); + + matches!( + normalized.as_str(), + "password" + | "passphrase" + | "mnemonic" + | "seed" + | "seedphrase" + | "seedwords" + | "jwt" + | "authorization" + | "bearer" + | "apikey" + | "accesstoken" + | "refreshtoken" + | "rpcauth" + | "rpcurl" + ) || normalized.contains("privatekey") + || normalized.contains("secret") + || normalized.contains("keyshare") + || normalized.contains("plaintext") + || normalized.contains("decrypted") + || normalized.contains("credential") + || normalized.contains("authorization") + || normalized.contains("jwt") + || normalized.contains("bearertoken") + || normalized.ends_with("apikey") + || normalized.ends_with("accesstoken") + || normalized.ends_with("refreshtoken") + || normalized.ends_with("rpcauth") + || normalized.ends_with("rpcurl") +} + +/// OpenTelemetry cannot rewrite a sibling layer's fields. Sensitive callsites +/// are therefore excluded from that exporter using their static field metadata. +pub fn telemetry_metadata_is_safe(metadata: &tracing::Metadata<'_>) -> bool { + !metadata + .fields() + .iter() + .any(|field| is_sensitive_field(field.name())) +} + /// Strip ANSI escape sequences (e.g. the terminal colours the event bus adds /// to log lines) so the structured store and JSONL file hold clean text. fn strip_ansi(input: &str) -> String { @@ -139,6 +192,11 @@ impl LogVisitor { .map(str::to_owned) .or_else(|| Some(value.to_string())); } else { + let value = if is_sensitive_field(field.name()) { + Value::String(REDACTED_FIELD_VALUE.to_owned()) + } else { + value + }; // Bound the field count so a deeply nested span tree can't grow an // entry without limit; always allow overwriting an existing key. if self.fields.len() >= MAX_FIELDS && !self.fields.contains_key(field.name()) { @@ -241,4 +299,79 @@ mod tests { ); assert_eq!(entry.message, ">>> ansi-marker-xyz payload"); } + + #[test] + fn redacts_sensitive_event_and_span_fields_before_storage() { + let collector = LogCollector::init("test-node", None); + let subscriber = tracing_subscriber::registry().with(OperationalLogLayer); + let sentinels = [ + "wallet-private-key-sentinel", + "keyshare-sentinel", + "password-sentinel", + "plaintext-sentinel", + "credential-sentinel", + "jwt-sentinel", + ]; + + tracing::subscriber::with_default(subscriber, || { + let span = tracing::info_span!( + "sensitive-span", + private_key = sentinels[0], + key_share = ?sentinels[1], + password = sentinels[2], + e3_id = "31337:11" + ); + let _guard = span.enter(); + tracing::info!( + decrypted_plaintext = sentinels[3], + signing_credentials = sentinels[4], + pinata_jwt = sentinels[5], + tx_hash = "0xsafe", + "sensitive redaction test" + ); + }); + + let result = collector.query(None, Some(10), None, None, Some("sensitive redaction test")); + let entry = result.entries.last().expect("captured tracing event"); + for field in [ + "private_key", + "key_share", + "password", + "decrypted_plaintext", + "signing_credentials", + "pinata_jwt", + ] { + assert_eq!(entry.fields[field], REDACTED_FIELD_VALUE); + } + assert_eq!(entry.fields["e3_id"], "31337:11"); + assert_eq!(entry.fields["tx_hash"], "0xsafe"); + + let serialized = serde_json::to_string(entry).expect("log entry must serialize"); + for sentinel in sentinels { + assert!( + !serialized.contains(sentinel), + "sensitive value leaked through field redaction: {sentinel}" + ); + } + } + + #[test] + fn sensitive_field_matching_handles_common_name_styles() { + for field in [ + "privateKey", + "SECRET_KEY", + "threshold-keyshare", + "raw_plaintext", + "rpc_url", + "access_token", + ] { + assert!(is_sensitive_field(field), "field was not redacted: {field}"); + } + for field in ["public_key", "tx_hash", "e3_id", "token_amount"] { + assert!( + !is_sensitive_field(field), + "safe field was redacted: {field}" + ); + } + } } From 8f35d14b2e365d582770c2aadbae1f45f9cb90d4 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 19:25:37 +0500 Subject: [PATCH 44/62] fix(evm): update typed-data ABI hashing --- Cargo.lock | 50 ++++++++++++++++++++++++++++++-------------------- Cargo.toml | 6 +++--- 2 files changed, 33 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index be01d98f7a..f74a19b708 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,7 +103,7 @@ dependencies = [ "derive_more", "encoding_rs", "flate2", - "foldhash", + "foldhash 0.1.5", "futures-core", "h2 0.3.27", "http 0.2.12", @@ -217,7 +217,7 @@ dependencies = [ "cookie", "derive_more", "encoding_rs", - "foldhash", + "foldhash 0.1.5", "futures-core", "futures-util", "impl-more", @@ -511,9 +511,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e8a436f0aad7df8bb47f144095fba61202265d9f5f09a70b0e3227881a668e" +checksum = "3fdff496dd4e98a81f4861e66f7eaf5f2488971848bb42d9c892f871730245c8" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -613,9 +613,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459f98c6843f208856f338bfb25e65325467f7aff35dfeb0484d0a76e059134b" +checksum = "5513d5e6bd1cba6bdcf5373470f559f320c05c8c59493b6e98912fbe6733943f" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -700,17 +700,17 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cfebde8c581a5d37b678d0a48a32decb51efd7a63a08ce2517ddec26db705c8" +checksum = "355bf68a433e0fd7f7d33d5a9fc2583fde70bf5c530f63b80845f8da5505cf28" dependencies = [ "alloy-rlp", "bytes", "cfg-if", "const-hex", "derive_more", - "foldhash", - "hashbrown 0.15.5", + "foldhash 0.2.0", + "hashbrown 0.16.1", "indexmap 2.13.0", "itoa", "k256", @@ -1002,9 +1002,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedac07a10d4c2027817a43cc1f038313fc53c7ac866f7363239971fd01f9f18" +checksum = "f3ce480400051b5217f19d6e9a82d9010cdde20f1ae9c00d53591e4a1afbb312" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -1016,9 +1016,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24f9a598f010f048d8b8226492b6401104f5a5c1273c2869b72af29b48bb4ba9" +checksum = "6d792e205ed3b72f795a8044c52877d2e6b6e9b1d13f431478121d8d4eaa9028" dependencies = [ "alloy-json-abi", "alloy-sol-macro-input", @@ -1035,9 +1035,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-input" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f494adf9d60e49aa6ce26dfd42c7417aa6d4343cf2ae621f20e4d92a5ad07d85" +checksum = "0bd1247a8f90b465ef3f1207627547ec16940c35597875cdc09c49d58b19693c" dependencies = [ "alloy-json-abi", "const-hex", @@ -1063,9 +1063,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a285b46e3e0c177887028278f04cc8262b76fd3b8e0e20e93cea0a58c35f5ac5" +checksum = "70319350969a3af119da6fb3e9bddb1bce66c9ea933600cb297c8b1850ad2a3c" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -4461,6 +4461,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -4833,8 +4839,7 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", - "serde", + "foldhash 0.1.5", ] [[package]] @@ -4842,6 +4847,11 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", + "serde", + "serde_core", +] [[package]] name = "heck" diff --git a/Cargo.toml b/Cargo.toml index d1935ba1bb..72b4a43cd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,13 +126,13 @@ actix = "=0.13.5" actix-web = "=4.11.0" aes-gcm = "=0.10.3" alloy = { version = "=1.0.41", features = ["node-bindings", "full"] } -alloy-primitives = { version = "=1.3.0", default-features = false, features = [ +alloy-primitives = { version = "=1.4.1", default-features = false, features = [ "rlp", "serde", "std", ] } -alloy-sol-types = "=1.3.0" -alloy-dyn-abi = "=1.3.0" +alloy-sol-types = "=1.4.1" +alloy-dyn-abi = "=1.4.1" argon2 = "=0.5.3" ark-ff = "0.5.0" ark-bn254 = "0.5.0" From 07420604c826eda76563023bbe90fba0b87ba0a9 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 20:02:20 +0500 Subject: [PATCH 45/62] fix(build): synchronize Rust consumer lockfiles --- Cargo.toml | 1 - examples/CRISP/Cargo.lock | 73 ++++++++++++++++++++++-------------- examples/CRISP/Cargo.toml | 6 +-- templates/default/Cargo.lock | 52 ++++++++++++++++--------- 4 files changed, 82 insertions(+), 50 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 72b4a43cd5..9918edae3c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,7 +56,6 @@ exclude = [ "templates/default" ] resolver = "3" -msrv = "1.91.1" [workspace.metadata.release] shared-version = true diff --git a/examples/CRISP/Cargo.lock b/examples/CRISP/Cargo.lock index 02a7c1baad..1075b459e9 100644 --- a/examples/CRISP/Cargo.lock +++ b/examples/CRISP/Cargo.lock @@ -77,7 +77,7 @@ dependencies = [ "derive_more", "encoding_rs", "flate2", - "foldhash", + "foldhash 0.1.5", "futures-core", "h2 0.3.27", "http 0.2.12", @@ -191,7 +191,7 @@ dependencies = [ "cookie", "derive_more", "encoding_rs", - "foldhash", + "foldhash 0.1.5", "futures-core", "futures-util", "impl-more", @@ -418,9 +418,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e8a436f0aad7df8bb47f144095fba61202265d9f5f09a70b0e3227881a668e" +checksum = "3fdff496dd4e98a81f4861e66f7eaf5f2488971848bb42d9c892f871730245c8" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -520,9 +520,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459f98c6843f208856f338bfb25e65325467f7aff35dfeb0484d0a76e059134b" +checksum = "5513d5e6bd1cba6bdcf5373470f559f320c05c8c59493b6e98912fbe6733943f" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -607,17 +607,17 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cfebde8c581a5d37b678d0a48a32decb51efd7a63a08ce2517ddec26db705c8" +checksum = "355bf68a433e0fd7f7d33d5a9fc2583fde70bf5c530f63b80845f8da5505cf28" dependencies = [ "alloy-rlp", "bytes", "cfg-if 1.0.4", "const-hex", "derive_more", - "foldhash", - "hashbrown 0.15.5", + "foldhash 0.2.0", + "hashbrown 0.16.1", "indexmap 2.14.0", "itoa", "k256", @@ -832,7 +832,7 @@ dependencies = [ "alloy-rlp", "alloy-serde", "alloy-sol-types", - "itertools 0.13.0", + "itertools 0.14.0", "serde", "serde_json", "serde_with", @@ -909,9 +909,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedac07a10d4c2027817a43cc1f038313fc53c7ac866f7363239971fd01f9f18" +checksum = "f3ce480400051b5217f19d6e9a82d9010cdde20f1ae9c00d53591e4a1afbb312" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", @@ -923,9 +923,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24f9a598f010f048d8b8226492b6401104f5a5c1273c2869b72af29b48bb4ba9" +checksum = "6d792e205ed3b72f795a8044c52877d2e6b6e9b1d13f431478121d8d4eaa9028" dependencies = [ "alloy-json-abi", "alloy-sol-macro-input", @@ -942,9 +942,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-input" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f494adf9d60e49aa6ce26dfd42c7417aa6d4343cf2ae621f20e4d92a5ad07d85" +checksum = "0bd1247a8f90b465ef3f1207627547ec16940c35597875cdc09c49d58b19693c" dependencies = [ "alloy-json-abi", "const-hex", @@ -970,9 +970,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a285b46e3e0c177887028278f04cc8262b76fd3b8e0e20e93cea0a58c35f5ac5" +checksum = "70319350969a3af119da6fb3e9bddb1bce66c9ea933600cb297c8b1850ad2a3c" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -1136,7 +1136,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1147,7 +1147,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -2587,6 +2587,7 @@ dependencies = [ "actix", "alloy", "anyhow", + "bincode", "derivative", "e3-utils-derive", "hex", @@ -2784,7 +2785,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2969,6 +2970,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -3270,8 +3277,18 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", "serde", + "serde_core", ] [[package]] @@ -4655,7 +4672,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -4674,7 +4691,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2 1.0.106", "quote 1.0.44", "syn 2.0.116", @@ -5207,7 +5224,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -5658,7 +5675,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -5820,7 +5837,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/examples/CRISP/Cargo.toml b/examples/CRISP/Cargo.toml index 9d026f0932..3cdc2a7b23 100644 --- a/examples/CRISP/Cargo.toml +++ b/examples/CRISP/Cargo.toml @@ -22,12 +22,12 @@ e3-user-program = { path = "./program" } crisp-utils = { path = "./crates/crisp-utils" } alloy = { version = "=1.0.41", features = ["full", "rpc-types-eth"] } -alloy-primitives = { version = "=1.3.0", default-features = false, features = [ +alloy-primitives = { version = "=1.4.1", default-features = false, features = [ "rlp", "serde", "std", ]} -alloy-sol-types = { version = "=1.3.0" } +alloy-sol-types = { version = "=1.4.1" } anyhow = { version = "=1.0.98" } actix-web = "=4.11.0" bincode = { version = "=1.3.3" } @@ -53,4 +53,4 @@ fhe-math = { git = "https://github.com/gnosisguild/fhe.rs", tag = "v0.2.2" } fhe-util = { git = "https://github.com/gnosisguild/fhe.rs", tag = "v0.2.2" } tokio = { version = "=1.46.1", features = ["full"] } rand = { version = "0.9" } -tracing-subscriber = { version = "=0.3.19", features = ["env-filter"] } +tracing-subscriber = { version = "=0.3.20", features = ["env-filter"] } diff --git a/templates/default/Cargo.lock b/templates/default/Cargo.lock index 238cbebde9..ec49083688 100644 --- a/templates/default/Cargo.lock +++ b/templates/default/Cargo.lock @@ -37,7 +37,7 @@ dependencies = [ "derive_more", "encoding_rs", "flate2", - "foldhash", + "foldhash 0.1.5", "futures-core", "h2 0.3.27", "http 0.2.12", @@ -151,7 +151,7 @@ dependencies = [ "cookie", "derive_more", "encoding_rs", - "foldhash", + "foldhash 0.1.5", "futures-core", "futures-util", "impl-more", @@ -253,9 +253,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alloy-dyn-abi" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9e8a436f0aad7df8bb47f144095fba61202265d9f5f09a70b0e3227881a668e" +checksum = "3fdff496dd4e98a81f4861e66f7eaf5f2488971848bb42d9c892f871730245c8" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -269,9 +269,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459f98c6843f208856f338bfb25e65325467f7aff35dfeb0484d0a76e059134b" +checksum = "5513d5e6bd1cba6bdcf5373470f559f320c05c8c59493b6e98912fbe6733943f" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -281,17 +281,17 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cfebde8c581a5d37b678d0a48a32decb51efd7a63a08ce2517ddec26db705c8" +checksum = "355bf68a433e0fd7f7d33d5a9fc2583fde70bf5c530f63b80845f8da5505cf28" dependencies = [ "alloy-rlp", "bytes", "cfg-if", "const-hex", "derive_more", - "foldhash", - "hashbrown 0.15.5", + "foldhash 0.2.0", + "hashbrown 0.16.1", "indexmap", "itoa", "k256", @@ -376,9 +376,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "1.3.0" +version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a285b46e3e0c177887028278f04cc8262b76fd3b8e0e20e93cea0a58c35f5ac5" +checksum = "70319350969a3af119da6fb3e9bddb1bce66c9ea933600cb297c8b1850ad2a3c" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -1553,7 +1553,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1719,6 +1719,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1908,8 +1914,18 @@ checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" +dependencies = [ + "foldhash 0.2.0", "serde", + "serde_core", ] [[package]] @@ -2962,7 +2978,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -2981,7 +2997,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -3388,7 +3404,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3841,7 +3857,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] From 64d44b1b60a12b8d49b72a897a26e9eed4d527a8 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 20:28:18 +0500 Subject: [PATCH 46/62] fix(sync): allow atomic identity bootstrap before schema stamp --- crates/data/src/data_store.rs | 39 +++++++++ crates/data/src/in_mem.rs | 10 ++- crates/data/src/in_mem_kv_store.rs | 27 +++++++ crates/data/src/sled_db.rs | 40 +++++++++ crates/data/src/sled_store.rs | 11 ++- crates/entrypoint/src/validate.rs | 11 ++- crates/sync/src/actors/sync.rs | 126 +++++++++++++++++++++++++++-- 7 files changed, 250 insertions(+), 14 deletions(-) diff --git a/crates/data/src/data_store.rs b/crates/data/src/data_store.rs index 4dab90a245..4711d71a05 100644 --- a/crates/data/src/data_store.rs +++ b/crates/data/src/data_store.rs @@ -37,6 +37,29 @@ pub struct ShutdownStore; #[rtype(result = "Result")] pub(crate) struct StoreIsEmpty; +#[derive(Message, Debug)] +#[rtype(result = "Result")] +pub(crate) struct StoreHasExactKeys { + keys: Vec>, +} + +impl StoreHasExactKeys { + fn new(keys: I) -> Self + where + K: IntoKey, + I: IntoIterator, + { + let mut keys = keys.into_iter().map(IntoKey::into_key).collect::>(); + keys.sort(); + keys.dedup(); + Self { keys } + } + + pub(crate) fn keys(&self) -> &[Vec] { + &self.keys + } +} + impl StoreAddr { pub fn to_maybe_in_mem(&self) -> Option<&Addr> { match self { @@ -75,6 +98,22 @@ impl DataStore { } } + /// Return whether the complete backing store contains exactly `keys` and no others. + /// + /// This intentionally ignores the current scope. Callers use it for whole-store safety + /// checks where allowing an unknown persisted key would be unsafe. + pub async fn has_exact_keys(&self, keys: I) -> Result + where + K: IntoKey, + I: IntoIterator, + { + let message = StoreHasExactKeys::new(keys); + match &self.addr { + StoreAddr::InMem(store) => Ok(store.send(message).await??), + StoreAddr::Sled(store) => Ok(store.send(message).await??), + } + } + /// Read data at the scope location pub async fn read(&self) -> Result> where diff --git a/crates/data/src/in_mem.rs b/crates/data/src/in_mem.rs index e398e1e025..14b28a0469 100644 --- a/crates/data/src/in_mem.rs +++ b/crates/data/src/in_mem.rs @@ -6,7 +6,7 @@ use crate::{ in_mem_kv_store::{DataOp, InMemKvStore}, - ShutdownStore, StoreIsEmpty, + ShutdownStore, StoreHasExactKeys, StoreIsEmpty, }; use actix::{Actor, ActorContext, Handler, Message}; use anyhow::Result; @@ -83,6 +83,14 @@ impl Handler for InMemStore { } } +impl Handler for InMemStore { + type Result = Result; + + fn handle(&mut self, message: StoreHasExactKeys, _: &mut Self::Context) -> Self::Result { + Ok(self.store.has_exact_keys(message.keys())) + } +} + impl Handler for InMemStore { type Result = Result; diff --git a/crates/data/src/in_mem_kv_store.rs b/crates/data/src/in_mem_kv_store.rs index 144009eeff..0002103c1b 100644 --- a/crates/data/src/in_mem_kv_store.rs +++ b/crates/data/src/in_mem_kv_store.rs @@ -73,6 +73,15 @@ impl InMemKvStore { self.db.entries().is_empty() } + /// Return whether the store contains exactly the supplied keys and no others. + pub fn has_exact_keys(&self, keys: &[Vec]) -> bool { + let entries = self.db.entries(); + entries.len() == keys.len() + && keys + .iter() + .all(|key| entries.iter().any(|(entry_key, _)| entry_key == key)) + } + /// Returns the captured operation log. pub fn log(&self) -> Vec { self.log.clone() @@ -128,6 +137,24 @@ mod tests { assert_eq!(Some(b"v2".to_vec()), store.get(b"k")); } + #[test] + fn exact_key_match_rejects_missing_and_extra_keys() { + let mut store = InMemKvStore::new(false); + store.insert(b"identity-a".to_vec(), b"1".to_vec(), None); + store.insert(b"identity-b".to_vec(), b"2".to_vec(), None); + + assert!(store.has_exact_keys(&[b"identity-a".to_vec(), b"identity-b".to_vec()])); + assert!(!store.has_exact_keys(&[b"identity-a".to_vec()])); + assert!(!store.has_exact_keys(&[ + b"identity-a".to_vec(), + b"identity-b".to_vec(), + b"protocol-state".to_vec() + ])); + + store.insert(b"protocol-state".to_vec(), b"3".to_vec(), None); + assert!(!store.has_exact_keys(&[b"identity-a".to_vec(), b"identity-b".to_vec()])); + } + #[test] fn capture_records_ops_only_when_enabled() { let insert = Insert::new(b"k".to_vec(), b"v".to_vec()); diff --git a/crates/data/src/sled_db.rs b/crates/data/src/sled_db.rs index e3a0562182..2bc7ee0cfb 100644 --- a/crates/data/src/sled_db.rs +++ b/crates/data/src/sled_db.rs @@ -91,6 +91,28 @@ impl SledDb { self.db.is_empty() } + /// Return whether the tree contains exactly the supplied keys and no others. + pub fn has_exact_keys(&self, keys: &[Vec]) -> Result { + let mut expected = keys.to_vec(); + expected.sort(); + expected.dedup(); + + // `Tree::len` walks the full tree. Schema preflight must remain bounded even when an + // unversioned store is unexpectedly large, so inspect at most one key beyond the allowed + // set and fail closed on any iterator error. + let observed = self + .db + .iter() + .keys() + .take(expected.len() + 1) + .collect::>>()?; + Ok(observed.len() == expected.len() + && observed + .iter() + .zip(expected) + .all(|(observed, expected)| observed.as_ref() == expected.as_slice())) + } + pub fn flush(&self) -> Result<()> { self.db.flush()?; Ok(()) @@ -230,4 +252,22 @@ mod tests { assert_eq!(db.get(Get::new("missing"))?, None); Ok(()) } + + #[test] + fn exact_key_match_rejects_missing_and_extra_keys() -> Result<()> { + use tempfile::tempdir; + + let temp_dir = tempdir()?; + let db_path = temp_dir.path().join("test_exact_keys.db"); + let mut db = SledDb::new(&db_path, "datastore")?; + db.insert(Insert::new("identity-a", b"1".to_vec()))?; + db.insert(Insert::new("identity-b", b"2".to_vec()))?; + + assert!(db.has_exact_keys(&[b"identity-a".to_vec(), b"identity-b".to_vec()])?); + assert!(!db.has_exact_keys(&[b"identity-a".to_vec()])?); + + db.insert(Insert::new("protocol-state", b"3".to_vec()))?; + assert!(!db.has_exact_keys(&[b"identity-a".to_vec(), b"identity-b".to_vec()])?); + Ok(()) + } } diff --git a/crates/data/src/sled_store.rs b/crates/data/src/sled_store.rs index 9099320ac0..abfb78975c 100644 --- a/crates/data/src/sled_store.rs +++ b/crates/data/src/sled_store.rs @@ -4,7 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use crate::{ShutdownStore, SledDb, StoreIsEmpty}; +use crate::{ShutdownStore, SledDb, StoreHasExactKeys, StoreIsEmpty}; use actix::{Actor, ActorContext, Addr, Handler, ResponseFuture}; use anyhow::{Context, Result}; use e3_events::{BusHandle, EType, ErrorDispatcher, Flush, InterfoldEvent, Unsequenced}; @@ -156,6 +156,15 @@ impl Handler for SledStore { } } +impl Handler for SledStore { + type Result = Result; + + fn handle(&mut self, message: StoreHasExactKeys, _: &mut Self::Context) -> Self::Result { + let db = self.db.as_ref().context("SledStore is closed")?; + db.has_exact_keys(message.keys()) + } +} + impl Handler for SledStore { type Result = Result<()>; fn handle(&mut self, _: Flush, _: &mut Self::Context) -> Self::Result { diff --git a/crates/entrypoint/src/validate.rs b/crates/entrypoint/src/validate.rs index bc712ef39b..9519b08277 100644 --- a/crates/entrypoint/src/validate.rs +++ b/crates/entrypoint/src/validate.rs @@ -41,7 +41,8 @@ use e3_events::{ }; use e3_sortition::{committee_key, NodeRegistry, NodeStateRepositoryFactory, NodeStateStore}; use e3_sync::{ - decide_schema_version, SchemaVersionDecision, SyncRepositoryFactory, SCHEMA_VERSION, + decide_schema_version, has_schema_governed_kv_state, SchemaVersionDecision, + SyncRepositoryFactory, SCHEMA_VERSION, }; use e3_utils::enumerate_path; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -210,7 +211,8 @@ pub async fn validate_node(config: &AppConfig) -> Result { // framing and decode checks. Cross-check each persisted replay cursor. let repositories = get_repositories(config)?; let persisted_schema = repositories.schema_version().read().await?; - let has_existing_state = total_events > 0 || !repositories.store.is_empty().await?; + let has_existing_state = + total_events > 0 || has_schema_governed_kv_state(&repositories).await?; report.push(check_schema_compatibility( persisted_schema, has_existing_state, @@ -236,8 +238,9 @@ pub async fn validate_node(config: &AppConfig) -> Result { } /// Verify that this binary can safely interpret the persisted schema. A missing -/// marker is acceptable only for a genuinely empty store; stamping a version on -/// non-empty legacy bytes would assert compatibility without evidence. +/// marker is acceptable only for a fresh store (empty or containing the complete bootstrap +/// identity pair); stamping a version on protocol or unknown bytes would assert compatibility +/// without evidence. fn check_schema_compatibility(persisted: Option, has_existing_state: bool) -> CheckResult { let name = "schema"; match decide_schema_version(persisted, SCHEMA_VERSION, has_existing_state) { diff --git a/crates/sync/src/actors/sync.rs b/crates/sync/src/actors/sync.rs index 8d8bd1a75a..db9770f06d 100644 --- a/crates/sync/src/actors/sync.rs +++ b/crates/sync/src/actors/sync.rs @@ -16,7 +16,7 @@ use e3_events::{ AggregateConfig, BusHandle, CorrelationId, EffectsEnabled, Event, EventContextAccessors, EventPublisher, EventStoreQueryBy, EventStoreQueryResponse, EventSubscriber, EventType, EvmEventConfig, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, HistoricalNetSyncStart, - InterfoldEvent, InterfoldEventData, SeqAgg, SyncEnded, Unsequenced, + InterfoldEvent, InterfoldEventData, SeqAgg, StoreKeys, SyncEnded, Unsequenced, }; use e3_utils::actix::channel as actix_toolbox; use std::time::Duration; @@ -207,9 +207,9 @@ fn sort_replay_events(events: &mut [InterfoldEvent]) { /// Validate or initialize the durable schema marker before runtime actors can write state. /// -/// A fresh empty store is stamped synchronously; incompatible or unversioned non-empty state -/// halts. The check is idempotent so composition roots may run it before actor construction while -/// [`sync`] retains the same guard for direct callers. +/// A fresh store is stamped synchronously; incompatible or unversioned protocol state halts. The +/// check is idempotent so composition roots may run it before actor construction while [`sync`] +/// retains the same guard for direct callers. pub async fn preflight_schema_version( repositories: &Repositories, aggregate_config: &AggregateConfig, @@ -218,7 +218,7 @@ pub async fn preflight_schema_version( let repo = repositories.schema_version(); let persisted = repo.read().await?; let has_existing_state = if persisted.is_none() { - !repositories.store.is_empty().await? + has_schema_governed_kv_state(repositories).await? || event_logs_have_events(aggregate_config, eventstore).await? } else { false @@ -236,6 +236,25 @@ pub async fn preflight_schema_version( } } +/// Return whether the key/value store contains state whose interpretation requires a schema +/// marker. +/// +/// Wallet setup runs before node startup and atomically writes the encrypted operator and libp2p +/// identities. That complete pair is the only pre-schema key/value state considered fresh. A +/// partial identity, any additional key, or any other non-empty key set remains schema-governed so +/// startup fails closed instead of asserting compatibility with unknown bytes. +pub async fn has_schema_governed_kv_state(repositories: &Repositories) -> Result { + if repositories.store.is_empty().await? { + return Ok(false); + } + + let bootstrap_identity_keys = [StoreKeys::eth_private_key(), StoreKeys::libp2p_keypair()]; + Ok(!repositories + .store + .has_exact_keys(bootstrap_identity_keys) + .await?) +} + async fn event_logs_have_events( aggregate_config: &AggregateConfig, eventstore: &Recipient>, @@ -316,17 +335,18 @@ impl SnapshotLoaded { #[cfg(test)] mod tests { use super::{ - collect_historical_evm_events, preflight_schema_version, publish_reconciled_history, - seed_clock_from_replay, sort_replay_events, + collect_historical_evm_events, has_schema_governed_kv_state, preflight_schema_version, + publish_reconciled_history, seed_clock_from_replay, sort_replay_events, }; use crate::domain::SyncPlanner; + use crate::SyncRepositoryFactory; use e3_ciphernode_builder::EventSystem; use e3_data::Repositories; use e3_events::{ hlc::{Hlc, HlcTimestamp}, EffectsEnabled, Event, EventPublisher, EventSubscriber, EventType, EvmEventConfig, EvmEventConfigChain, GetEvents, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, - InterfoldEvent, InterfoldEventData, SyncEnded, TakeEvents, Unsequenced, + InterfoldEvent, InterfoldEventData, StoreKeys, SyncEnded, TakeEvents, Unsequenced, }; use std::collections::BTreeMap; @@ -374,6 +394,96 @@ mod tests { Ok(()) } + #[actix::test] + async fn schema_preflight_initializes_store_with_only_bootstrap_identity() -> anyhow::Result<()> + { + let system = EventSystem::new().with_fresh_bus(); + let store = system.store()?; + let private_key = vec![1_u8, 2, 3]; + let network_key = vec![4_u8, 5, 6]; + store + .write_batch_sync([ + (StoreKeys::eth_private_key(), private_key.clone()), + (StoreKeys::libp2p_keypair(), network_key.clone()), + ]) + .await?; + let repositories = Repositories::from(&store); + let eventstore = system.eventstore_reader()?.seq(); + + assert!(!has_schema_governed_kv_state(&repositories).await?); + preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore).await?; + + assert_eq!( + repositories.schema_version().read().await?, + Some(super::SCHEMA_VERSION) + ); + assert_eq!( + store + .scope(StoreKeys::eth_private_key()) + .read::>() + .await?, + Some(private_key) + ); + assert_eq!( + store + .scope(StoreKeys::libp2p_keypair()) + .read::>() + .await?, + Some(network_key) + ); + Ok(()) + } + + #[actix::test] + async fn schema_preflight_rejects_bootstrap_identity_plus_protocol_state() -> anyhow::Result<()> + { + let system = EventSystem::new().with_fresh_bus(); + let store = system.store()?; + store + .write_batch_sync([ + (StoreKeys::eth_private_key(), vec![1_u8]), + (StoreKeys::libp2p_keypair(), vec![2_u8]), + ]) + .await?; + store + .scope(StoreKeys::sortition()) + .write_sync(7_u64) + .await?; + let repositories = Repositories::from(&store); + let eventstore = system.eventstore_reader()?.seq(); + + assert!(has_schema_governed_kv_state(&repositories).await?); + let error = + preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) + .await + .unwrap_err(); + + assert!(error.to_string().contains("no schema marker")); + assert_eq!(repositories.schema_version().read().await?, None); + Ok(()) + } + + #[actix::test] + async fn schema_preflight_rejects_partial_bootstrap_identity() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let store = system.store()?; + store + .scope(StoreKeys::eth_private_key()) + .write_sync(vec![1_u8]) + .await?; + let repositories = Repositories::from(&store); + let eventstore = system.eventstore_reader()?.seq(); + + let error = + preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) + .await + .unwrap_err(); + + assert!(error.to_string().contains("no schema marker")); + assert_eq!(repositories.schema_version().read().await?, None); + Ok(()) + } + #[actix::test] async fn schema_preflight_rejects_unversioned_event_log() -> anyhow::Result<()> { let system = EventSystem::new().with_fresh_bus(); From c96ac0c7571a9d275ba73fe50f7a72ebd4c88991 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 20:34:44 +0500 Subject: [PATCH 47/62] fix(dev): wire program server authentication to callers --- examples/CRISP/Readme.md | 5 ++ examples/CRISP/crisp.dev.env.example | 8 +++ examples/CRISP/scripts/dev_program.sh | 9 ++- examples/CRISP/scripts/dev_server.sh | 10 ++- examples/CRISP/scripts/lib/dev_config.sh | 14 ++++- examples/CRISP/server/.env.example | 2 + examples/CRISP/server/src/config.rs | 31 +++++++++- .../src/server/program_server_request.rs | 62 ++++++++++++++++--- templates/default/README.md | 6 ++ templates/default/package.json | 3 +- .../default/scripts/dev_all_concurrently.sh | 3 +- templates/default/scripts/dev_program.sh | 8 +++ templates/default/scripts/dev_server.sh | 7 +++ templates/default/scripts/lib/dev_config.sh | 7 +++ templates/default/server/runner.test.ts | 56 +++++++++++++++++ templates/default/server/runner.ts | 8 +-- templates/default/server/utils.ts | 11 +++- 17 files changed, 226 insertions(+), 24 deletions(-) create mode 100644 templates/default/server/runner.test.ts diff --git a/examples/CRISP/Readme.md b/examples/CRISP/Readme.md index 25f29d3854..c78df9c0c9 100644 --- a/examples/CRISP/Readme.md +++ b/examples/CRISP/Readme.md @@ -60,6 +60,11 @@ pnpm dev:setup pnpm dev:up ``` +The local profile also contains the shared, non-secret development program-server token and its +allowed callback origin. The dev launch scripts pass that token to both the authenticated +development program server and the coordination server. Outside this local profile, both +components require an operator-managed `INTERFOLD_PROGRAM_SERVER_TOKEN`; neither has a fallback. + `dev:up` runs `scripts/dev.sh`, which: 1. Starts the Hardhat node in `packages/crisp-contracts` diff --git a/examples/CRISP/crisp.dev.env.example b/examples/CRISP/crisp.dev.env.example index 7ffe9a9635..4273a8a9fa 100644 --- a/examples/CRISP/crisp.dev.env.example +++ b/examples/CRISP/crisp.dev.env.example @@ -15,3 +15,11 @@ CRISP_BFV_PRESET=insecure-512 # - server/.env gets E3_PROOF_AGGREGATION_ENABLED=true # When false (default): mock verifiers, no DKG circuit build, faster DKG. CRISP_PROOF_AGGREGATION_ENABLED=false + +# Shared authentication for the local CRISP coordination server and development program server. +# This value is intentionally non-secret and is only loaded by CRISP's local dev scripts. Set a +# strong operator-managed INTERFOLD_PROGRAM_SERVER_TOKEN for any non-local deployment. +INTERFOLD_PROGRAM_SERVER_TOKEN=crisp-local-development-only + +# The development program server accepts callbacks only to this exact origin. +INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN=http://127.0.0.1:4000 diff --git a/examples/CRISP/scripts/dev_program.sh b/examples/CRISP/scripts/dev_program.sh index 2b84cb6db2..091a230d85 100755 --- a/examples/CRISP/scripts/dev_program.sh +++ b/examples/CRISP/scripts/dev_program.sh @@ -1,6 +1,11 @@ - #!/usr/bin/env bash -set -e +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/dev_config.sh" + +load_crisp_dev_config interfold program start --dev true diff --git a/examples/CRISP/scripts/dev_server.sh b/examples/CRISP/scripts/dev_server.sh index 3e1aa838bf..e1893a4549 100755 --- a/examples/CRISP/scripts/dev_server.sh +++ b/examples/CRISP/scripts/dev_server.sh @@ -1,7 +1,13 @@ #!/usr/bin/env bash -set -e +set -euo pipefail export CARGO_INCREMENTAL=1 -(cd ./server && rm -rf database && cargo run --bin server) +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/dev_config.sh" + +load_crisp_dev_config + +(cd "${CRISP_ROOT}/server" && rm -rf database && cargo run --bin server) diff --git a/examples/CRISP/scripts/lib/dev_config.sh b/examples/CRISP/scripts/lib/dev_config.sh index a96f0038e3..b6ecdcd6e2 100644 --- a/examples/CRISP/scripts/lib/dev_config.sh +++ b/examples/CRISP/scripts/lib/dev_config.sh @@ -23,6 +23,16 @@ load_crisp_dev_config() { CRISP_BFV_PRESET="${CRISP_BFV_PRESET:-insecure-512}" CRISP_PROOF_AGGREGATION_ENABLED="${CRISP_PROOF_AGGREGATION_ENABLED:-false}" + if [[ -z "${INTERFOLD_PROGRAM_SERVER_TOKEN:-}" ]]; then + echo "INTERFOLD_PROGRAM_SERVER_TOKEN must be set in ${cfg}" >&2 + return 1 + fi + + if [[ -z "${INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN:-}" ]]; then + echo "INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN must be set in ${cfg}" >&2 + return 1 + fi + case "$CRISP_BFV_PRESET" in insecure-512 | secure-8192) ;; *) @@ -46,6 +56,7 @@ load_crisp_dev_config() { fi export CRISP_BFV_PRESET CRISP_PROOF_AGGREGATION_ENABLED CRISP_ROOT REPO_ROOT + export INTERFOLD_PROGRAM_SERVER_TOKEN INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN } _set_env_kv() { @@ -103,9 +114,10 @@ CRISP dev profile (${CRISP_ROOT}/crisp.dev.env): CRISP_BFV_PRESET=${CRISP_BFV_PRESET} CRISP_PROOF_AGGREGATION_ENABLED=${CRISP_PROOF_AGGREGATION_ENABLED} ENABLE_ZK_VERIFICATION=${ENABLE_ZK_VERIFICATION:-false} (used at deploy via dev:up) + INTERFOLD_PROGRAM_SERVER_TOKEN=configured (value redacted) + INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN=${INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN} server/.env E3_PROOF_AGGREGATION_ENABLED synced by dev:setup Contract addresses synced by dev:up (deploy → server/.env, client/.env, interfold.config.yaml) EOF } - diff --git a/examples/CRISP/server/.env.example b/examples/CRISP/server/.env.example index cb1b3fe2a1..7c8ca16d0e 100644 --- a/examples/CRISP/server/.env.example +++ b/examples/CRISP/server/.env.example @@ -4,6 +4,8 @@ PRIVATE_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 INTERFOLD_SERVER_URL=http://127.0.0.1:4000 HTTP_RPC_URL=http://127.0.0.1:8545 PROGRAM_SERVER_URL=http://127.0.0.1:13151 +# Local dev scripts load INTERFOLD_PROGRAM_SERVER_TOKEN from ../crisp.dev.env. Direct and +# non-local launches must provide an operator-managed value in their process environment. WS_RPC_URL=ws://127.0.0.1:8545 CHAIN_ID=31337 diff --git a/examples/CRISP/server/src/config.rs b/examples/CRISP/server/src/config.rs index 12678e98ae..06a18dd02e 100644 --- a/examples/CRISP/server/src/config.rs +++ b/examples/CRISP/server/src/config.rs @@ -9,9 +9,11 @@ use dotenvy::dotenv; use once_cell::sync::Lazy; use serde::Deserialize; -#[derive(Debug, Deserialize)] +// Do not derive `Debug`: this structure owns private keys and service credentials. +#[derive(Deserialize)] pub struct Config { pub program_server_url: String, + pub interfold_program_server_token: String, pub interfold_server_url: String, pub private_key: String, pub http_rpc_url: String, @@ -37,6 +39,15 @@ pub struct Config { pub etherscan_api_key: String, } +fn validate_program_server_token(token: &str) -> Result<(), ConfigError> { + if token.trim().is_empty() { + return Err(ConfigError::Message( + "INTERFOLD_PROGRAM_SERVER_TOKEN must not be empty".to_string(), + )); + } + Ok(()) +} + impl Config { /// Base URL for outbound HTTP clients (program-server webhooks, CLI, cron). /// @@ -56,12 +67,26 @@ impl Config { } else { dotenv().ok(); } - ConfigManager::builder() + let config: Self = ConfigManager::builder() .add_source(config::Environment::default()) .build()? - .try_deserialize() + .try_deserialize()?; + validate_program_server_token(&config.interfold_program_server_token)?; + Ok(config) } } pub static CONFIG: Lazy = Lazy::new(|| Config::from_env().expect("Failed to load configuration")); + +#[cfg(test)] +mod tests { + use super::validate_program_server_token; + + #[test] + fn program_server_token_must_be_nonempty() { + assert!(validate_program_server_token("operator-token").is_ok()); + assert!(validate_program_server_token("").is_err()); + assert!(validate_program_server_token(" ").is_err()); + } +} diff --git a/examples/CRISP/server/src/server/program_server_request.rs b/examples/CRISP/server/src/server/program_server_request.rs index b4cf6e08d2..1b3bbcd353 100644 --- a/examples/CRISP/server/src/server/program_server_request.rs +++ b/examples/CRISP/server/src/server/program_server_request.rs @@ -44,6 +44,18 @@ pub struct ProcessingResponse { pub e3_id: u64, } +fn build_compute_request( + client: &reqwest::Client, + program_server_url: &str, + bearer_token: &str, + request: &ComputeRequest, +) -> reqwest::RequestBuilder { + client + .post(format!("{program_server_url}/run_compute")) + .bearer_auth(bearer_token) + .json(request) +} + pub async fn run_compute( e3_id: u64, params: Vec, @@ -59,13 +71,49 @@ pub async fn run_compute( println!("Sending request"); - let response: ProcessingResponse = reqwest::Client::new() - .post(format!("{}/{}", CONFIG.program_server_url, "run_compute")) - .json(&request) - .send() - .await? - .json() - .await?; + let response: ProcessingResponse = build_compute_request( + &reqwest::Client::new(), + &CONFIG.program_server_url, + &CONFIG.interfold_program_server_token, + &request, + ) + .send() + .await? + .error_for_status()? + .json() + .await?; Ok((response.e3_id, response.status)) } + +#[cfg(test)] +mod tests { + use reqwest::header::AUTHORIZATION; + + use super::{build_compute_request, ComputeRequest}; + + #[test] + fn compute_request_uses_configured_bearer_token() { + let request = ComputeRequest { + e3_id: Some(7), + params: vec![1, 2, 3], + ciphertext_inputs: vec![], + callback_url: Some("http://127.0.0.1:4000/state/add-result".to_string()), + }; + + let request = build_compute_request( + &reqwest::Client::new(), + "http://127.0.0.1:13151", + "local-test-token", + &request, + ) + .build() + .expect("request should build"); + + assert_eq!(request.url().as_str(), "http://127.0.0.1:13151/run_compute"); + assert_eq!( + request.headers().get(AUTHORIZATION).unwrap(), + "Bearer local-test-token" + ); + } +} diff --git a/templates/default/README.md b/templates/default/README.md index ca85d4109d..f37075ce77 100644 --- a/templates/default/README.md +++ b/templates/default/README.md @@ -120,6 +120,12 @@ Launch all services with one command: pnpm dev:all ``` +The local dev scripts provide one shared, non-secret credential to the program server and the +TypeScript coordination server, and restrict callbacks to the local coordination-server origin. +Running the authenticated development program server or TypeScript coordinator outside these +scripts requires operators to set `INTERFOLD_PROGRAM_SERVER_TOKEN`; the program server also +requires `INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN`. Neither component supplies a fallback. + This starts: - Local Ethereum network (Hardhat) diff --git a/templates/default/package.json b/templates/default/package.json index 7d62a1d179..6dfebfa89a 100644 --- a/templates/default/package.json +++ b/templates/default/package.json @@ -19,7 +19,8 @@ "dev:server": "./scripts/dev_server.sh", "predev:all": "[ ! -f './.interfold/generated/contracts/ImageID.sol' ] && interfold program compile || true", "test": "hardhat test", - "test:integration": "./scripts/test_integration.sh" + "test:server": "vitest run ./server/runner.test.ts", + "test:integration": "pnpm test:server && ./scripts/test_integration.sh" }, "dependencies": { "@interfold/contracts": "workspace:*", diff --git a/templates/default/scripts/dev_all_concurrently.sh b/templates/default/scripts/dev_all_concurrently.sh index c6d2e6d3c9..31ea769a47 100755 --- a/templates/default/scripts/dev_all_concurrently.sh +++ b/templates/default/scripts/dev_all_concurrently.sh @@ -18,5 +18,4 @@ pnpm concurrently \ "anvil --host 0.0.0.0 --chain-id 31337 --block-time 1 --mnemonic 'test test test test test test test test test test test junk' --silent" \ "pnpm dev:ciphernodes" \ "TEST_MODE=1 pnpm dev:server" \ - "interfold program start" - + "pnpm dev:program" diff --git a/templates/default/scripts/dev_program.sh b/templates/default/scripts/dev_program.sh index a409b429a2..395dfdcac7 100755 --- a/templates/default/scripts/dev_program.sh +++ b/templates/default/scripts/dev_program.sh @@ -1,6 +1,14 @@ #!/usr/bin/env bash set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/dev_config.sh" + +load_template_dev_config +cd "${TEMPLATE_ROOT}" + echo "interfold rev = $(interfold rev)" echo "Waiting on ciphernodes to be ready..." pnpm wait-on file:/tmp/interfold_ciphernodes_ready && interfold program start diff --git a/templates/default/scripts/dev_server.sh b/templates/default/scripts/dev_server.sh index 8586e56ed7..433fed07ad 100755 --- a/templates/default/scripts/dev_server.sh +++ b/templates/default/scripts/dev_server.sh @@ -2,6 +2,13 @@ set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib/dev_config.sh" + +load_template_dev_config +cd "${TEMPLATE_ROOT}" + echo "Waiting for evm node..." pnpm wait-on tcp:localhost:8545 echo "Waiting for contracts to be deployed..." diff --git a/templates/default/scripts/lib/dev_config.sh b/templates/default/scripts/lib/dev_config.sh index 2bce2bf45c..77a3f02f8d 100644 --- a/templates/default/scripts/lib/dev_config.sh +++ b/templates/default/scripts/lib/dev_config.sh @@ -12,6 +12,12 @@ load_template_dev_config() { BFV_PRESET="${BFV_PRESET:-insecure-512}" COMMITTEE="${COMMITTEE:-minimum}" + # These defaults are intentionally limited to the template's local development scripts. The + # authenticated development program server and TypeScript caller require operators to provide a + # credential when launched outside this local orchestration. + INTERFOLD_PROGRAM_SERVER_TOKEN="${INTERFOLD_PROGRAM_SERVER_TOKEN:-template-local-development-only}" + INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN="${INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN:-http://127.0.0.1:8080}" + case "$BFV_PRESET" in insecure-512 | secure-8192) ;; *) @@ -21,6 +27,7 @@ load_template_dev_config() { esac export TEMPLATE_ROOT INTERFOLD_REPO_ROOT BFV_PRESET COMMITTEE + export INTERFOLD_PROGRAM_SERVER_TOKEN INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN } template_monorepo_build_available() { diff --git a/templates/default/server/runner.test.ts b/templates/default/server/runner.test.ts new file mode 100644 index 0000000000..d6195f2fe4 --- /dev/null +++ b/templates/default/server/runner.test.ts @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest' + +import { callFheRunner } from './runner' + +describe('callFheRunner', () => { + beforeEach(() => { + delete process.env.INTERFOLD_PROGRAM_SERVER_TOKEN + delete process.env.PROGRAM_RUNNER_URL + delete process.env.CALLBACK_URL + }) + + afterEach(() => { + vi.restoreAllMocks() + vi.unstubAllGlobals() + }) + + it('requires an operator-provided token outside local dev orchestration', async () => { + await expect(callFheRunner(7n, '0x01', [])).rejects.toThrow('Missing required env var: INTERFOLD_PROGRAM_SERVER_TOKEN') + }) + + it('authenticates the request without logging the token or payload', async () => { + const token = 'focused-runner-test-token' + process.env.INTERFOLD_PROGRAM_SERVER_TOKEN = token + process.env.PROGRAM_RUNNER_URL = 'http://127.0.0.1:13151' + process.env.CALLBACK_URL = 'http://127.0.0.1:8080' + + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ status: 'processing', e3_id: 7 }), + }) + vi.stubGlobal('fetch', fetchMock) + const logMock = vi.spyOn(console, 'log').mockImplementation(() => undefined) + + await callFheRunner(7n, '0x0102', [['0x03', 0]]) + + expect(fetchMock).toHaveBeenCalledOnce() + expect(fetchMock).toHaveBeenCalledWith( + 'http://127.0.0.1:13151/run_compute', + expect.objectContaining({ + headers: { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }, + }), + ) + expect(JSON.stringify(logMock.mock.calls)).not.toContain(token) + expect(JSON.stringify(logMock.mock.calls)).not.toContain('0x0102') + expect(JSON.stringify(logMock.mock.calls)).not.toContain('0x03') + }) +}) diff --git a/templates/default/server/runner.ts b/templates/default/server/runner.ts index b3253ad830..0a462e4518 100644 --- a/templates/default/server/runner.ts +++ b/templates/default/server/runner.ts @@ -4,10 +4,10 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -import { getCheckedEnvVars } from './utils' +import { getProgramRunnerConfig } from './utils' export async function callFheRunner(e3Id: bigint, params: string, ciphertextInputs: Array<[string, number]>): Promise { - const { PROGRAM_RUNNER_URL, CALLBACK_URL } = getCheckedEnvVars() + const { PROGRAM_RUNNER_URL, CALLBACK_URL, PROGRAM_SERVER_TOKEN } = getProgramRunnerConfig() const payload = { e3_id: Number(e3Id), @@ -15,12 +15,12 @@ export async function callFheRunner(e3Id: bigint, params: string, ciphertextInpu ciphertext_inputs: ciphertextInputs, callback_url: CALLBACK_URL, } - console.log('payload:') - console.log(JSON.stringify(payload)) + console.log(`Submitting E3 ${e3Id} to FHE runner with ${ciphertextInputs.length} ciphertext input(s)`) const response = await fetch(`${PROGRAM_RUNNER_URL}/run_compute`, { method: 'POST', headers: { + Authorization: `Bearer ${PROGRAM_SERVER_TOKEN}`, 'Content-Type': 'application/json', }, body: JSON.stringify(payload), diff --git a/templates/default/server/utils.ts b/templates/default/server/utils.ts index 421ce5176a..b874b9045c 100644 --- a/templates/default/server/utils.ts +++ b/templates/default/server/utils.ts @@ -12,6 +12,14 @@ export function ensureEnv(key: string): string { return value } +export function getProgramRunnerConfig() { + return { + PROGRAM_RUNNER_URL: process.env.PROGRAM_RUNNER_URL || 'http://127.0.0.1:13151', + CALLBACK_URL: process.env.CALLBACK_URL || 'http://127.0.0.1:8080', + PROGRAM_SERVER_TOKEN: ensureEnv('INTERFOLD_PROGRAM_SERVER_TOKEN'), + } +} + export function getCheckedEnvVars() { return { RPC_URL: ensureEnv('RPC_URL'), @@ -21,7 +29,6 @@ export function getCheckedEnvVars() { FEE_TOKEN_CONTRACT: ensureEnv('FEE_TOKEN_ADDRESS'), PRIVATE_KEY: ensureEnv('PRIVATE_KEY'), CHAIN_ID: parseInt(ensureEnv('CHAIN_ID')), - PROGRAM_RUNNER_URL: process.env.PROGRAM_RUNNER_URL || 'http://127.0.0.1:13151', - CALLBACK_URL: process.env.CALLBACK_URL || 'http://127.0.0.1:8080', + ...getProgramRunnerConfig(), } } From b090ae607b4d89215894bb00e442361f8e20c392 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 21:33:38 +0500 Subject: [PATCH 48/62] revert(program-server): remove development bearer auth --- crates/program-server/src/lib.rs | 148 ++---------------- crates/support-scripts/dev/src/main.rs | 8 +- examples/CRISP/Readme.md | 7 +- examples/CRISP/crisp.dev.env.example | 8 - examples/CRISP/scripts/dev_server.sh | 7 +- examples/CRISP/scripts/lib/dev_config.sh | 13 -- examples/CRISP/server/.env.example | 2 - examples/CRISP/server/src/config.rs | 30 +--- .../src/server/program_server_request.rs | 22 +-- templates/default/README.md | 8 +- templates/default/scripts/dev_server.sh | 6 +- templates/default/scripts/lib/dev_config.sh | 7 - templates/default/server/runner.test.ts | 11 +- templates/default/server/runner.ts | 3 +- templates/default/server/utils.ts | 1 - 15 files changed, 34 insertions(+), 247 deletions(-) diff --git a/crates/program-server/src/lib.rs b/crates/program-server/src/lib.rs index a57d8490b6..70b1368747 100644 --- a/crates/program-server/src/lib.rs +++ b/crates/program-server/src/lib.rs @@ -6,10 +6,7 @@ mod types; -use actix_web::{ - http::header, middleware::Logger, web, App, HttpRequest, HttpResponse, HttpServer, - Result as ActixResult, -}; +use actix_web::{middleware::Logger, web, App, HttpResponse, HttpServer, Result as ActixResult}; use anyhow::{Context, Result}; use e3_compute_provider::FHEInputs; use serde::Serialize; @@ -32,8 +29,6 @@ pub struct E3ProgramServerBuilder { port: Option, host: Option, localhost_rewrite: Option, - bearer_token: Option, - callback_origin: Option, max_concurrent_jobs: usize, } @@ -49,8 +44,6 @@ impl E3ProgramServerBuilder { port: None, host: None, localhost_rewrite: None, - bearer_token: None, - callback_origin: None, max_concurrent_jobs: 1, } } @@ -73,18 +66,6 @@ impl E3ProgramServerBuilder { self } - /// Require this bearer token on every compute request. - pub fn with_bearer_token(mut self, token: impl Into) -> Self { - self.bearer_token = Some(token.into()); - self - } - - /// Allow callbacks only to this exact URL origin (scheme, host, and port). - pub fn with_callback_origin(mut self, origin: impl Into) -> Self { - self.callback_origin = Some(origin.into()); - self - } - /// Bound the number of computations that may execute concurrently. pub fn with_max_concurrent_jobs(mut self, max_concurrent_jobs: usize) -> Self { self.max_concurrent_jobs = max_concurrent_jobs; @@ -93,20 +74,6 @@ impl E3ProgramServerBuilder { /// Build the E3ProgramServer pub fn build(self) -> Result { - let bearer_token = self - .bearer_token - .filter(|token| !token.is_empty()) - .context("program server requires a non-empty bearer token")?; - let callback_origin = self - .callback_origin - .context("program server requires an allowed callback origin")?; - let callback_origin = parse_http_url(&callback_origin, "callback origin")?; - anyhow::ensure!( - callback_origin.path() == "/" - && callback_origin.query().is_none() - && callback_origin.fragment().is_none(), - "callback origin must not contain a path, query, or fragment" - ); anyhow::ensure!( self.max_concurrent_jobs > 0, "max concurrent jobs must be greater than zero" @@ -122,8 +89,6 @@ impl E3ProgramServerBuilder { port: self.port.unwrap_or(13151), host: self.host.unwrap_or_else(|| "0.0.0.0".to_string()), localhost_rewrite: self.localhost_rewrite, - bearer_token: Arc::from(bearer_token), - callback_origin, webhook_client, jobs: Arc::new(Semaphore::new(self.max_concurrent_jobs)), }) @@ -136,8 +101,6 @@ pub struct E3ProgramServer { port: u16, host: String, localhost_rewrite: Option, - bearer_token: Arc, - callback_origin: reqwest::Url, webhook_client: reqwest::Client, jobs: Arc, } @@ -173,8 +136,6 @@ impl E3ProgramServer { let config = AppConfig { runner: Arc::clone(&self.runner), localhost_rewrite: self.localhost_rewrite.clone(), - bearer_token: Arc::clone(&self.bearer_token), - callback_origin: self.callback_origin.clone(), webhook_client: self.webhook_client.clone(), jobs: Arc::clone(&self.jobs), }; @@ -198,37 +159,10 @@ impl E3ProgramServer { pub struct AppConfig { pub runner: Arc, pub localhost_rewrite: Option, - bearer_token: Arc, - callback_origin: reqwest::Url, webhook_client: reqwest::Client, jobs: Arc, } -fn constant_time_eq(left: &[u8], right: &[u8]) -> bool { - let mut difference = left.len() ^ right.len(); - let max_len = left.len().max(right.len()); - for index in 0..max_len { - difference |= left.get(index).copied().unwrap_or(0) as usize - ^ right.get(index).copied().unwrap_or(0) as usize; - } - difference == 0 -} - -fn authorize_compute(request: &HttpRequest, expected_token: &str) -> ActixResult<()> { - let supplied = request - .headers() - .get(header::AUTHORIZATION) - .and_then(|value| value.to_str().ok()) - .and_then(|value| value.strip_prefix("Bearer ")); - if supplied.is_some_and(|token| constant_time_eq(token.as_bytes(), expected_token.as_bytes())) { - Ok(()) - } else { - Err(actix_web::error::ErrorUnauthorized( - "missing or invalid bearer token", - )) - } -} - fn parse_http_url(value: &str, label: &str) -> Result { let url = reqwest::Url::parse(value).with_context(|| format!("invalid {label}"))?; anyhow::ensure!( @@ -246,7 +180,6 @@ fn parse_http_url(value: &str, label: &str) -> Result { fn validated_callback_url( callback_url: &str, localhost_rewrite: Option<&str>, - allowed_origin: &reqwest::Url, ) -> Result { let mut callback = parse_http_url(callback_url, "callback URL")?; if matches!(callback.host_str(), Some("localhost" | "127.0.0.1")) { @@ -256,12 +189,6 @@ fn validated_callback_url( .map_err(|_| anyhow::anyhow!("invalid localhost rewrite host"))?; } } - anyhow::ensure!( - callback.scheme() == allowed_origin.scheme() - && callback.host_str() == allowed_origin.host_str() - && callback.port_or_known_default() == allowed_origin.port_or_known_default(), - "callback URL origin is not allowed" - ); anyhow::ensure!( callback.fragment().is_none(), "callback URL must not contain a fragment" @@ -361,10 +288,8 @@ async fn process_computation_background( async fn handle_compute( config: web::Data, - request: HttpRequest, req: web::Json, ) -> ActixResult { - authorize_compute(&request, &config.bearer_token)?; println!("Processing computation..."); let e3_id = req .e3_id @@ -380,12 +305,8 @@ async fn handle_compute( ciphertexts: req.ciphertext_inputs.clone(), }; - let callback_url = validated_callback_url( - &callback_url, - config.localhost_rewrite.as_deref(), - &config.callback_origin, - ) - .map_err(actix_web::error::ErrorBadRequest)?; + let callback_url = validated_callback_url(&callback_url, config.localhost_rewrite.as_deref()) + .map_err(actix_web::error::ErrorBadRequest)?; let permit = Arc::clone(&config.jobs) .try_acquire_owned() .map_err(|_| actix_web::error::ErrorTooManyRequests("compute capacity exhausted"))?; @@ -417,45 +338,24 @@ async fn handle_health_check() -> ActixResult { #[cfg(test)] mod server_tests { use super::*; - use actix_web::test::TestRequest; - - #[test] - fn builder_requires_a_nonempty_bearer_token() { - let missing = E3ProgramServer::builder(|_| async { Ok((vec![], vec![])) }).build(); - assert!(missing.is_err()); - - let empty = E3ProgramServer::builder(|_| async { Ok((vec![], vec![])) }) - .with_bearer_token("") - .build(); - assert!(empty.is_err()); - } #[test] - fn callback_validation_rejects_other_origins_and_unsafe_schemes() { - let allowed = reqwest::Url::parse("https://callback.example:8443").unwrap(); - assert!( - validated_callback_url("https://callback.example:8443/results/1", None, &allowed) - .is_ok() - ); - assert!( - validated_callback_url("https://metadata.internal:8443/latest", None, &allowed) - .is_err() - ); - assert!(validated_callback_url("file:///etc/passwd", None, &allowed).is_err()); + fn callback_validation_accepts_http_origins_and_rejects_unsafe_urls() { + assert!(validated_callback_url("https://callback.example:8443/results/1", None).is_ok()); + assert!(validated_callback_url("https://metadata.internal:8443/latest", None).is_ok()); + assert!(validated_callback_url("file:///etc/passwd", None).is_err()); + assert!(validated_callback_url("https://user:pass@example.com/result", None).is_err()); + assert!(validated_callback_url("https://example.com/result#fragment", None).is_err()); } #[test] fn builder_rejects_zero_capacity_and_configures_the_job_limit() { let zero = E3ProgramServer::builder(|_| async { Ok((vec![], vec![])) }) - .with_bearer_token("secret") - .with_callback_origin("https://callback.example") .with_max_concurrent_jobs(0) .build(); assert!(zero.is_err()); let server = E3ProgramServer::builder(|_| async { Ok((vec![], vec![])) }) - .with_bearer_token("secret") - .with_callback_origin("https://callback.example") .with_max_concurrent_jobs(2) .build() .unwrap(); @@ -464,32 +364,12 @@ mod server_tests { #[test] fn localhost_rewrite_changes_only_an_exact_local_host() { - let allowed = reqwest::Url::parse("http://host.local:8080").unwrap(); let rewritten = - validated_callback_url("http://127.0.0.1:8080/result", Some("host.local"), &allowed) - .unwrap(); + validated_callback_url("http://127.0.0.1:8080/result", Some("host.local")).unwrap(); assert_eq!(rewritten.as_str(), "http://host.local:8080/result"); - assert!(validated_callback_url( - "http://localhost.attacker:8080/result", - Some("host.local"), - &allowed - ) - .is_err()); - } - - #[test] - fn compute_authorization_accepts_only_the_configured_bearer_token() { - let missing = TestRequest::default().to_http_request(); - assert!(authorize_compute(&missing, "secret").is_err()); - - let wrong = TestRequest::default() - .insert_header((header::AUTHORIZATION, "Bearer wrong")) - .to_http_request(); - assert!(authorize_compute(&wrong, "secret").is_err()); - - let valid = TestRequest::default() - .insert_header((header::AUTHORIZATION, "Bearer secret")) - .to_http_request(); - assert!(authorize_compute(&valid, "secret").is_ok()); + let unchanged = + validated_callback_url("http://localhost.attacker:8080/result", Some("host.local")) + .unwrap(); + assert_eq!(unchanged.as_str(), "http://localhost.attacker:8080/result"); } } diff --git a/crates/support-scripts/dev/src/main.rs b/crates/support-scripts/dev/src/main.rs index 66d6a39b37..5ba2697c0d 100644 --- a/crates/support-scripts/dev/src/main.rs +++ b/crates/support-scripts/dev/src/main.rs @@ -4,24 +4,18 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::{Context, Result}; +use anyhow::Result; use e3_program_server::E3ProgramServer; use e3_user_program::fhe_processor; #[tokio::main] async fn main() -> Result<()> { - let bearer_token = std::env::var("INTERFOLD_PROGRAM_SERVER_TOKEN") - .context("INTERFOLD_PROGRAM_SERVER_TOKEN must be set")?; - let callback_origin = std::env::var("INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN") - .context("INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN must be set")?; let server = E3ProgramServer::builder(|inputs| async move { Ok(( vec![3, 1, 4, 1, 5, 9, 2, 6, 5, 3, 5], fhe_processor(&inputs), )) }) - .with_bearer_token(bearer_token) - .with_callback_origin(callback_origin) .build()?; server.run().await?; diff --git a/examples/CRISP/Readme.md b/examples/CRISP/Readme.md index c78df9c0c9..ec0622da8c 100644 --- a/examples/CRISP/Readme.md +++ b/examples/CRISP/Readme.md @@ -60,10 +60,9 @@ pnpm dev:setup pnpm dev:up ``` -The local profile also contains the shared, non-secret development program-server token and its -allowed callback origin. The dev launch scripts pass that token to both the authenticated -development program server and the coordination server. Outside this local profile, both -components require an operator-managed `INTERFOLD_PROGRAM_SERVER_TOKEN`; neither has a fallback. +The program server accepts caller-supplied HTTP(S) callback URLs. It is a development-only test +service, does not authenticate callers or allowlist callback destinations, and must stay isolated +from production and untrusted networks. `dev:up` runs `scripts/dev.sh`, which: diff --git a/examples/CRISP/crisp.dev.env.example b/examples/CRISP/crisp.dev.env.example index 4273a8a9fa..7ffe9a9635 100644 --- a/examples/CRISP/crisp.dev.env.example +++ b/examples/CRISP/crisp.dev.env.example @@ -15,11 +15,3 @@ CRISP_BFV_PRESET=insecure-512 # - server/.env gets E3_PROOF_AGGREGATION_ENABLED=true # When false (default): mock verifiers, no DKG circuit build, faster DKG. CRISP_PROOF_AGGREGATION_ENABLED=false - -# Shared authentication for the local CRISP coordination server and development program server. -# This value is intentionally non-secret and is only loaded by CRISP's local dev scripts. Set a -# strong operator-managed INTERFOLD_PROGRAM_SERVER_TOKEN for any non-local deployment. -INTERFOLD_PROGRAM_SERVER_TOKEN=crisp-local-development-only - -# The development program server accepts callbacks only to this exact origin. -INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN=http://127.0.0.1:4000 diff --git a/examples/CRISP/scripts/dev_server.sh b/examples/CRISP/scripts/dev_server.sh index e1893a4549..51268d1a5e 100755 --- a/examples/CRISP/scripts/dev_server.sh +++ b/examples/CRISP/scripts/dev_server.sh @@ -5,9 +5,4 @@ set -euo pipefail export CARGO_INCREMENTAL=1 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck disable=SC1091 -source "${SCRIPT_DIR}/lib/dev_config.sh" - -load_crisp_dev_config - -(cd "${CRISP_ROOT}/server" && rm -rf database && cargo run --bin server) +(cd "${SCRIPT_DIR}/../server" && rm -rf database && cargo run --bin server) diff --git a/examples/CRISP/scripts/lib/dev_config.sh b/examples/CRISP/scripts/lib/dev_config.sh index b6ecdcd6e2..19a003b5a1 100644 --- a/examples/CRISP/scripts/lib/dev_config.sh +++ b/examples/CRISP/scripts/lib/dev_config.sh @@ -23,16 +23,6 @@ load_crisp_dev_config() { CRISP_BFV_PRESET="${CRISP_BFV_PRESET:-insecure-512}" CRISP_PROOF_AGGREGATION_ENABLED="${CRISP_PROOF_AGGREGATION_ENABLED:-false}" - if [[ -z "${INTERFOLD_PROGRAM_SERVER_TOKEN:-}" ]]; then - echo "INTERFOLD_PROGRAM_SERVER_TOKEN must be set in ${cfg}" >&2 - return 1 - fi - - if [[ -z "${INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN:-}" ]]; then - echo "INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN must be set in ${cfg}" >&2 - return 1 - fi - case "$CRISP_BFV_PRESET" in insecure-512 | secure-8192) ;; *) @@ -56,7 +46,6 @@ load_crisp_dev_config() { fi export CRISP_BFV_PRESET CRISP_PROOF_AGGREGATION_ENABLED CRISP_ROOT REPO_ROOT - export INTERFOLD_PROGRAM_SERVER_TOKEN INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN } _set_env_kv() { @@ -114,8 +103,6 @@ CRISP dev profile (${CRISP_ROOT}/crisp.dev.env): CRISP_BFV_PRESET=${CRISP_BFV_PRESET} CRISP_PROOF_AGGREGATION_ENABLED=${CRISP_PROOF_AGGREGATION_ENABLED} ENABLE_ZK_VERIFICATION=${ENABLE_ZK_VERIFICATION:-false} (used at deploy via dev:up) - INTERFOLD_PROGRAM_SERVER_TOKEN=configured (value redacted) - INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN=${INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN} server/.env E3_PROOF_AGGREGATION_ENABLED synced by dev:setup Contract addresses synced by dev:up (deploy → server/.env, client/.env, interfold.config.yaml) diff --git a/examples/CRISP/server/.env.example b/examples/CRISP/server/.env.example index 7c8ca16d0e..cb1b3fe2a1 100644 --- a/examples/CRISP/server/.env.example +++ b/examples/CRISP/server/.env.example @@ -4,8 +4,6 @@ PRIVATE_KEY=0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 INTERFOLD_SERVER_URL=http://127.0.0.1:4000 HTTP_RPC_URL=http://127.0.0.1:8545 PROGRAM_SERVER_URL=http://127.0.0.1:13151 -# Local dev scripts load INTERFOLD_PROGRAM_SERVER_TOKEN from ../crisp.dev.env. Direct and -# non-local launches must provide an operator-managed value in their process environment. WS_RPC_URL=ws://127.0.0.1:8545 CHAIN_ID=31337 diff --git a/examples/CRISP/server/src/config.rs b/examples/CRISP/server/src/config.rs index 06a18dd02e..56c421bfe6 100644 --- a/examples/CRISP/server/src/config.rs +++ b/examples/CRISP/server/src/config.rs @@ -9,11 +9,10 @@ use dotenvy::dotenv; use once_cell::sync::Lazy; use serde::Deserialize; -// Do not derive `Debug`: this structure owns private keys and service credentials. +// Do not derive `Debug`: this structure owns private keys and other secrets. #[derive(Deserialize)] pub struct Config { pub program_server_url: String, - pub interfold_program_server_token: String, pub interfold_server_url: String, pub private_key: String, pub http_rpc_url: String, @@ -39,15 +38,6 @@ pub struct Config { pub etherscan_api_key: String, } -fn validate_program_server_token(token: &str) -> Result<(), ConfigError> { - if token.trim().is_empty() { - return Err(ConfigError::Message( - "INTERFOLD_PROGRAM_SERVER_TOKEN must not be empty".to_string(), - )); - } - Ok(()) -} - impl Config { /// Base URL for outbound HTTP clients (program-server webhooks, CLI, cron). /// @@ -67,26 +57,12 @@ impl Config { } else { dotenv().ok(); } - let config: Self = ConfigManager::builder() + ConfigManager::builder() .add_source(config::Environment::default()) .build()? - .try_deserialize()?; - validate_program_server_token(&config.interfold_program_server_token)?; - Ok(config) + .try_deserialize() } } pub static CONFIG: Lazy = Lazy::new(|| Config::from_env().expect("Failed to load configuration")); - -#[cfg(test)] -mod tests { - use super::validate_program_server_token; - - #[test] - fn program_server_token_must_be_nonempty() { - assert!(validate_program_server_token("operator-token").is_ok()); - assert!(validate_program_server_token("").is_err()); - assert!(validate_program_server_token(" ").is_err()); - } -} diff --git a/examples/CRISP/server/src/server/program_server_request.rs b/examples/CRISP/server/src/server/program_server_request.rs index 1b3bbcd353..af1a21390f 100644 --- a/examples/CRISP/server/src/server/program_server_request.rs +++ b/examples/CRISP/server/src/server/program_server_request.rs @@ -47,12 +47,10 @@ pub struct ProcessingResponse { fn build_compute_request( client: &reqwest::Client, program_server_url: &str, - bearer_token: &str, request: &ComputeRequest, ) -> reqwest::RequestBuilder { client .post(format!("{program_server_url}/run_compute")) - .bearer_auth(bearer_token) .json(request) } @@ -74,7 +72,6 @@ pub async fn run_compute( let response: ProcessingResponse = build_compute_request( &reqwest::Client::new(), &CONFIG.program_server_url, - &CONFIG.interfold_program_server_token, &request, ) .send() @@ -93,7 +90,7 @@ mod tests { use super::{build_compute_request, ComputeRequest}; #[test] - fn compute_request_uses_configured_bearer_token() { + fn compute_request_does_not_require_authorization() { let request = ComputeRequest { e3_id: Some(7), params: vec![1, 2, 3], @@ -101,19 +98,12 @@ mod tests { callback_url: Some("http://127.0.0.1:4000/state/add-result".to_string()), }; - let request = build_compute_request( - &reqwest::Client::new(), - "http://127.0.0.1:13151", - "local-test-token", - &request, - ) - .build() - .expect("request should build"); + let request = + build_compute_request(&reqwest::Client::new(), "http://127.0.0.1:13151", &request) + .build() + .expect("request should build"); assert_eq!(request.url().as_str(), "http://127.0.0.1:13151/run_compute"); - assert_eq!( - request.headers().get(AUTHORIZATION).unwrap(), - "Bearer local-test-token" - ); + assert!(!request.headers().contains_key(AUTHORIZATION)); } } diff --git a/templates/default/README.md b/templates/default/README.md index f37075ce77..5cfc52e32c 100644 --- a/templates/default/README.md +++ b/templates/default/README.md @@ -120,11 +120,9 @@ Launch all services with one command: pnpm dev:all ``` -The local dev scripts provide one shared, non-secret credential to the program server and the -TypeScript coordination server, and restrict callbacks to the local coordination-server origin. -Running the authenticated development program server or TypeScript coordinator outside these -scripts requires operators to set `INTERFOLD_PROGRAM_SERVER_TOKEN`; the program server also -requires `INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN`. Neither component supplies a fallback. +The program server accepts caller-supplied HTTP(S) callback URLs. It is a development-only test +service, does not authenticate callers or allowlist callback destinations, and must stay isolated +from production and untrusted networks. This starts: diff --git a/templates/default/scripts/dev_server.sh b/templates/default/scripts/dev_server.sh index 433fed07ad..90ce49310d 100755 --- a/templates/default/scripts/dev_server.sh +++ b/templates/default/scripts/dev_server.sh @@ -3,11 +3,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck disable=SC1091 -source "${SCRIPT_DIR}/lib/dev_config.sh" - -load_template_dev_config -cd "${TEMPLATE_ROOT}" +cd "${SCRIPT_DIR}/.." echo "Waiting for evm node..." pnpm wait-on tcp:localhost:8545 diff --git a/templates/default/scripts/lib/dev_config.sh b/templates/default/scripts/lib/dev_config.sh index 77a3f02f8d..2bce2bf45c 100644 --- a/templates/default/scripts/lib/dev_config.sh +++ b/templates/default/scripts/lib/dev_config.sh @@ -12,12 +12,6 @@ load_template_dev_config() { BFV_PRESET="${BFV_PRESET:-insecure-512}" COMMITTEE="${COMMITTEE:-minimum}" - # These defaults are intentionally limited to the template's local development scripts. The - # authenticated development program server and TypeScript caller require operators to provide a - # credential when launched outside this local orchestration. - INTERFOLD_PROGRAM_SERVER_TOKEN="${INTERFOLD_PROGRAM_SERVER_TOKEN:-template-local-development-only}" - INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN="${INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN:-http://127.0.0.1:8080}" - case "$BFV_PRESET" in insecure-512 | secure-8192) ;; *) @@ -27,7 +21,6 @@ load_template_dev_config() { esac export TEMPLATE_ROOT INTERFOLD_REPO_ROOT BFV_PRESET COMMITTEE - export INTERFOLD_PROGRAM_SERVER_TOKEN INTERFOLD_PROGRAM_SERVER_CALLBACK_ORIGIN } template_monorepo_build_available() { diff --git a/templates/default/server/runner.test.ts b/templates/default/server/runner.test.ts index d6195f2fe4..2e7b40180d 100644 --- a/templates/default/server/runner.test.ts +++ b/templates/default/server/runner.test.ts @@ -10,7 +10,6 @@ import { callFheRunner } from './runner' describe('callFheRunner', () => { beforeEach(() => { - delete process.env.INTERFOLD_PROGRAM_SERVER_TOKEN delete process.env.PROGRAM_RUNNER_URL delete process.env.CALLBACK_URL }) @@ -20,13 +19,7 @@ describe('callFheRunner', () => { vi.unstubAllGlobals() }) - it('requires an operator-provided token outside local dev orchestration', async () => { - await expect(callFheRunner(7n, '0x01', [])).rejects.toThrow('Missing required env var: INTERFOLD_PROGRAM_SERVER_TOKEN') - }) - - it('authenticates the request without logging the token or payload', async () => { - const token = 'focused-runner-test-token' - process.env.INTERFOLD_PROGRAM_SERVER_TOKEN = token + it('submits without authorization and does not log the payload', async () => { process.env.PROGRAM_RUNNER_URL = 'http://127.0.0.1:13151' process.env.CALLBACK_URL = 'http://127.0.0.1:8080' @@ -44,12 +37,10 @@ describe('callFheRunner', () => { 'http://127.0.0.1:13151/run_compute', expect.objectContaining({ headers: { - Authorization: `Bearer ${token}`, 'Content-Type': 'application/json', }, }), ) - expect(JSON.stringify(logMock.mock.calls)).not.toContain(token) expect(JSON.stringify(logMock.mock.calls)).not.toContain('0x0102') expect(JSON.stringify(logMock.mock.calls)).not.toContain('0x03') }) diff --git a/templates/default/server/runner.ts b/templates/default/server/runner.ts index 0a462e4518..d0fcba297d 100644 --- a/templates/default/server/runner.ts +++ b/templates/default/server/runner.ts @@ -7,7 +7,7 @@ import { getProgramRunnerConfig } from './utils' export async function callFheRunner(e3Id: bigint, params: string, ciphertextInputs: Array<[string, number]>): Promise { - const { PROGRAM_RUNNER_URL, CALLBACK_URL, PROGRAM_SERVER_TOKEN } = getProgramRunnerConfig() + const { PROGRAM_RUNNER_URL, CALLBACK_URL } = getProgramRunnerConfig() const payload = { e3_id: Number(e3Id), @@ -20,7 +20,6 @@ export async function callFheRunner(e3Id: bigint, params: string, ciphertextInpu const response = await fetch(`${PROGRAM_RUNNER_URL}/run_compute`, { method: 'POST', headers: { - Authorization: `Bearer ${PROGRAM_SERVER_TOKEN}`, 'Content-Type': 'application/json', }, body: JSON.stringify(payload), diff --git a/templates/default/server/utils.ts b/templates/default/server/utils.ts index b874b9045c..82a776bbc1 100644 --- a/templates/default/server/utils.ts +++ b/templates/default/server/utils.ts @@ -16,7 +16,6 @@ export function getProgramRunnerConfig() { return { PROGRAM_RUNNER_URL: process.env.PROGRAM_RUNNER_URL || 'http://127.0.0.1:13151', CALLBACK_URL: process.env.CALLBACK_URL || 'http://127.0.0.1:8080', - PROGRAM_SERVER_TOKEN: ensureEnv('INTERFOLD_PROGRAM_SERVER_TOKEN'), } } From a206b9fad6b10d22307ce4a205e06f77f6ab02ce Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 21:37:23 +0500 Subject: [PATCH 49/62] fix(zk): restore committee slots before C6 verification --- .../06_DEACTIVATION_AND_COMPLETION.md | 9 + .../src/ciphernode_builder.rs | 45 ++++- crates/sortition/src/actors/sortition.rs | 29 ++- .../domain/finalized_committee_retention.rs | 71 ++++++++ crates/sortition/src/domain/mod.rs | 2 + crates/zk-prover/src/actors/mod.rs | 8 +- .../src/actors/share_verification.rs | 168 +++++++++++++++--- 7 files changed, 299 insertions(+), 33 deletions(-) create mode 100644 crates/sortition/src/domain/finalized_committee_retention.rs diff --git a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md index 5235fdd018..2785ca8b5b 100644 --- a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md +++ b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md @@ -218,6 +218,9 @@ On restart: │ 1. Load snapshot metadata and hydrate persisted per-E3 state │ → Extensions must preserve hydrated recipients; replayed committee events │ must not replace a restored per-E3 actor with a fresh instance +│ → ShareVerificationActor loads canonical party slots from the durable +│ finalized-committees repository before replay. A snapshotted +│ CommitteeFinalized event is not guaranteed to appear in the replay window. │ 2. CiphernodeSelector emits persisted AggregatorChanged state before replay │ → ThresholdPlaintextAggregatorExtension records this role in the E3 context │ so a plaintext buffer created later by CiphertextOutputPublished starts @@ -323,6 +326,12 @@ recovered `E3Meta` so restarted active aggregators can complete C6 verification. recipient, the restarted node can collect honest decryption shares and then wait forever for a consistency-check response that no actor is subscribed to publish. +The global `ShareVerificationActor` also requires the finalized committee's ordered party-slot map +for signer ownership checks. It is seeded from `Repositories::finalized_committees` during builder +startup, before EventStore replay. Relying only on a `CommitteeFinalized` subscription is incorrect: +once that event is included in a snapshot, replay starts after it and a restarted aggregator would +reject every honest C6 signer as having no canonical slot. + --- ## Rust-Side: E3 Lifecycle Coordinator (durable stage tracking) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 5807a354e3..a9d699cd9a 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -34,13 +34,13 @@ use e3_net::{ create_channel_bridge, setup_libp2p_keypair, setup_net_interface, setup_net_with_limits, NetRepositoryFactory, }; -use e3_request::E3LifecycleCoordinator; use e3_request::E3Router; +use e3_request::{E3LifecycleCoordinator, E3LifecycleRepositoryFactory}; use e3_slashing::{AccusationManagerExtension, CommitmentConsistencyCheckerExtension}; use e3_sortition::{ CiphernodeSelector, CiphernodeSelectorFactory, EmitPersistedAggregatorState, - FinalizedCommitteesRepositoryFactory, NodeStateRepositoryFactory, Sortition, SortitionBackend, - SortitionRepositoryFactory, + FinalizedCommitteeRetention, FinalizedCommitteesRepositoryFactory, NodeStateRepositoryFactory, + Sortition, SortitionBackend, SortitionRepositoryFactory, }; use e3_sync::{preflight_schema_version, sync}; use e3_utils::SharedRng; @@ -668,11 +668,26 @@ impl CiphernodeBuilder { ) -> Result<(Addr, Addr)> { let ciphernode_selector = CiphernodeSelector::attach(bus, repositories.ciphernode_selector(), addr).await?; + let committees_repo = repositories.finalized_committees(); + let mut committees = committees_repo.read().await?.unwrap_or_default(); + let lifecycle = repositories + .e3_lifecycle() + .read() + .await? + .unwrap_or_default(); + let pruned = FinalizedCommitteeRetention::prune_terminal(&mut committees, &lifecycle); + if pruned > 0 { + committees_repo.write_sync(&committees).await?; + info!( + pruned, + "Removed terminal finalized committees during startup" + ); + } let sortition = Sortition::attach( bus, repositories.sortition(), repositories.node_state(), - repositories.finalized_committees(), + committees_repo, self.sortition_backend.clone(), ciphernode_selector.clone(), addr, @@ -761,6 +776,12 @@ impl CiphernodeBuilder { accusation_vote_validity_by_chain: &HashMap, ) -> Result { let mut e3_builder = E3Router::builder(bus, store.clone()); + let persisted_committees = store + .repositories() + .finalized_committees() + .read() + .await? + .unwrap_or_default(); // ── Threshold keyshare + ZK actors ── if let Some(KeyshareKind::Threshold) = self.keyshare { @@ -777,7 +798,13 @@ impl CiphernodeBuilder { e3_builder.with(ThresholdKeyshareExtension::create(bus, &self.cipher, addr)); info!("Setting up ZK actors"); - setup_zk_actors(bus, backend, _signer, dkg_fold_verifier_by_chain.clone()); + setup_zk_actors( + bus, + backend, + _signer, + dkg_fold_verifier_by_chain.clone(), + persisted_committees.clone(), + ); } // ── Public key aggregation ── @@ -796,7 +823,13 @@ impl CiphernodeBuilder { .ok_or_else(|| anyhow::anyhow!("ZK backend is required for aggregator"))?; let signer = provider_cache.ensure_signer().await?; info!("Setting up ZK actors for aggregator"); - setup_zk_actors(bus, backend, signer, dkg_fold_verifier_by_chain.clone()); + setup_zk_actors( + bus, + backend, + signer, + dkg_fold_verifier_by_chain.clone(), + persisted_committees, + ); } } diff --git a/crates/sortition/src/actors/sortition.rs b/crates/sortition/src/actors/sortition.rs index b4532ee479..bdb0972300 100644 --- a/crates/sortition/src/actors/sortition.rs +++ b/crates/sortition/src/actors/sortition.rs @@ -12,13 +12,14 @@ use crate::messages::{ GetCommitteeMembersRequest, WithSortitionTicket, }; use crate::CiphernodeSelector; +use crate::FinalizedCommitteeRetention; use actix::prelude::*; use anyhow::{anyhow, Result}; use e3_data::{AutoPersist, Persistable, Repository}; use e3_events::{ prelude::*, trap, CiphernodeAdded, CiphernodeRemoved, Committee, CommitteeFinalized, - CommitteeMemberExpelled, CommitteePublished, ConfigurationUpdated, E3Failed, E3Requested, - E3Stage, E3StageChanged, EType, EventContext, EventType, InterfoldEvent, + CommitteeMemberExpelled, CommitteePublished, ConfigurationUpdated, E3Failed, E3RequestComplete, + E3Requested, E3Stage, E3StageChanged, EType, EventContext, EventType, InterfoldEvent, OperatorActivationChanged, PlaintextOutputPublished, Seed, Sequenced, TicketBalanceUpdated, TypedEvent, }; @@ -120,6 +121,7 @@ impl Sortition { EventType::CommitteeMemberExpelled, EventType::E3Failed, EventType::E3StageChanged, + EventType::E3RequestComplete, ], addr.clone().into(), ); @@ -295,6 +297,9 @@ impl Handler for Sortition { InterfoldEventData::E3StageChanged(data) => { self.notify_sync(ctx, TypedEvent::new(data, ec)) } + InterfoldEventData::E3RequestComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } _ => (), } } @@ -579,6 +584,26 @@ impl Handler> for Sortition { } } +impl Handler> for Sortition { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap(EType::Sortition, &self.bus.with_ec(msg.get_ctx()), || { + self.finalized_committees + .try_mutate(msg.get_ctx(), |mut committees| { + FinalizedCommitteeRetention::remove(&mut committees, &msg.e3_id); + Ok(committees) + })?; + self.pending_expulsions.remove(&msg.e3_id); + Ok(()) + }) + } +} + impl Handler> for Sortition { type Result = (); diff --git a/crates/sortition/src/domain/finalized_committee_retention.rs b/crates/sortition/src/domain/finalized_committee_retention.rs new file mode 100644 index 0000000000..fd0fb2af04 --- /dev/null +++ b/crates/sortition/src/domain/finalized_committee_retention.rs @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use e3_events::{Committee, E3Stage, E3id}; +use std::collections::HashMap; + +/// Pure lifecycle policy for restart-critical finalized committee state. +pub struct FinalizedCommitteeRetention; + +impl FinalizedCommitteeRetention { + pub fn remove(committees: &mut HashMap, e3_id: &E3id) -> bool { + committees.remove(e3_id).is_some() + } + + pub fn prune_terminal( + committees: &mut HashMap, + lifecycle: &HashMap, + ) -> usize { + let before = committees.len(); + committees.retain(|e3_id, _| { + !matches!( + lifecycle.get(e3_id), + Some(E3Stage::Complete | E3Stage::Failed) + ) + }); + before - committees.len() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn committee() -> Committee { + Committee::new(vec![]) + } + + #[test] + fn startup_pruning_removes_only_known_terminal_committees() { + let complete = E3id::new("1", 1); + let failed = E3id::new("2", 1); + let active = E3id::new("3", 1); + let unknown = E3id::new("4", 1); + let mut committees = HashMap::from([ + (complete.clone(), committee()), + (failed.clone(), committee()), + (active.clone(), committee()), + (unknown.clone(), committee()), + ]); + let lifecycle = HashMap::from([ + (complete.clone(), E3Stage::Complete), + (failed.clone(), E3Stage::Failed), + (active.clone(), E3Stage::KeyPublished), + ]); + + assert_eq!( + FinalizedCommitteeRetention::prune_terminal(&mut committees, &lifecycle), + 2 + ); + assert_eq!( + committees + .keys() + .cloned() + .collect::>(), + std::collections::HashSet::from([active, unknown]) + ); + } +} diff --git a/crates/sortition/src/domain/mod.rs b/crates/sortition/src/domain/mod.rs index 427a3a77e5..dd9f37bac8 100644 --- a/crates/sortition/src/domain/mod.rs +++ b/crates/sortition/src/domain/mod.rs @@ -5,11 +5,13 @@ // or FITNESS FOR A PARTICULAR PURPOSE. pub mod backends; +mod finalized_committee_retention; pub mod node_registry; pub mod ticket; pub mod ticket_sortition; pub use backends::*; +pub use finalized_committee_retention::*; pub use node_registry::*; pub use ticket::*; pub use ticket_sortition::*; diff --git a/crates/zk-prover/src/actors/mod.rs b/crates/zk-prover/src/actors/mod.rs index 66e76afeaa..fe0562d90a 100644 --- a/crates/zk-prover/src/actors/mod.rs +++ b/crates/zk-prover/src/actors/mod.rs @@ -24,13 +24,14 @@ //! use e3_zk_prover::{ZkBackend, setup_zk_actors}; //! use e3_events::BusHandle; //! use alloy::signers::local::PrivateKeySigner; +//! use std::collections::HashMap; //! //! let bus = BusHandle::default(); //! let backend = ZkBackend::with_default_dir().await?; //! let signer = PrivateKeySigner::random(); //! //! // Setup all actors with proper separation of concerns -//! setup_zk_actors(&bus, &backend, signer, HashMap::new()); +//! setup_zk_actors(&bus, &backend, signer, HashMap::new(), HashMap::new()); //! ``` pub mod accusation_manager; @@ -57,7 +58,7 @@ pub use zk_actor::ZkActor; use actix::{Actor, Addr}; use alloy::primitives::Address; use alloy::signers::local::PrivateKeySigner; -use e3_events::BusHandle; +use e3_events::{BusHandle, Committee, E3id}; use std::collections::HashMap; use crate::ZkBackend; @@ -74,13 +75,14 @@ pub fn setup_zk_actors( backend: &ZkBackend, signer: PrivateKeySigner, dkg_fold_attestation_verifiers_by_chain: HashMap>, + persisted_committees: HashMap, ) -> ZkActors { let zk_actor = ZkActor::new(backend).start(); let verifier = zk_actor.clone().recipient(); let proof_request = ProofRequestActor::setup(bus, signer.clone()); let proof_verification = ProofVerificationActor::setup(bus, verifier); - let share_verification = ShareVerificationActor::setup(bus); + let share_verification = ShareVerificationActor::setup(bus, persisted_committees); let node_proof_aggregator = NodeProofAggregator::setup(bus, signer, dkg_fold_attestation_verifiers_by_chain); diff --git a/crates/zk-prover/src/actors/share_verification.rs b/crates/zk-prover/src/actors/share_verification.rs index a37b20cbd5..1027af0ca7 100644 --- a/crates/zk-prover/src/actors/share_verification.rs +++ b/crates/zk-prover/src/actors/share_verification.rs @@ -27,7 +27,7 @@ use actix::{Actor, Addr, Context, Handler}; use alloy::primitives::{keccak256, Address, Bytes}; use alloy::sol_types::SolValue; use e3_events::{ - BusHandle, CommitmentConsistencyCheckComplete, CommitmentConsistencyCheckRequested, + BusHandle, CommitmentConsistencyCheckComplete, CommitmentConsistencyCheckRequested, Committee, ComputeRequest, ComputeRequestError, ComputeResponse, ComputeResponseKind, CorrelationId, E3id, EventContext, EventPublisher, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, PartyVerificationResult, ProofType, ProofVerificationFailed, ProofVerificationPassed, @@ -65,17 +65,21 @@ pub struct ShareVerificationActor { } impl ShareVerificationActor { - pub fn new(bus: &BusHandle) -> Self { - Self { + pub fn new(bus: &BusHandle, persisted_committees: HashMap) -> Self { + let mut actor = Self { bus: bus.clone(), committees: HashMap::new(), pending: HashMap::new(), pending_consistency: HashMap::new(), + }; + for (e3_id, committee) in persisted_committees { + actor.store_committee(e3_id, committee.members()); } + actor } - pub fn setup(bus: &BusHandle) -> Addr { - let addr = Self::new(bus).start(); + pub fn setup(bus: &BusHandle, persisted_committees: HashMap) -> Addr { + let addr = Self::new(bus, persisted_committees).start(); bus.subscribe(EventType::CommitteeFinalized, addr.clone().into()); bus.subscribe(EventType::ShareVerificationDispatched, addr.clone().into()); bus.subscribe(EventType::ComputeResponse, addr.clone().into()); @@ -88,6 +92,25 @@ impl ShareVerificationActor { addr } + fn store_committee(&mut self, e3_id: E3id, members: &[String]) { + match members + .iter() + .map(|member| member.parse()) + .collect::, _>>() + { + Ok(committee) => { + self.committees.insert(e3_id, committee); + } + Err(error) => { + error!( + %e3_id, + %error, + "Finalized committee contains an invalid address; share proofs will be rejected" + ); + } + } + } + fn handle_share_verification_dispatched( &mut self, msg: TypedEvent, @@ -595,23 +618,7 @@ impl Handler for ShareVerificationActor { // Mirror the C0 verifier's canonical ordering at this trust boundary. Replayed and // test-produced events are not assumed to have passed through the EVM decoder. data.sort_by_score(); - match data - .committee - .iter() - .map(|member| member.parse()) - .collect::, _>>() - { - Ok(committee) => { - self.committees.insert(data.e3_id, committee); - } - Err(error) => { - error!( - e3_id = %data.e3_id, - %error, - "Finalized committee contains an invalid address; share proofs will be rejected" - ); - } - } + self.store_committee(data.e3_id, &data.committee); } InterfoldEventData::ShareVerificationDispatched(data) => { self.notify_sync(ctx, TypedEvent::new(data, ec)) @@ -684,3 +691,120 @@ impl Handler> for ShareVerificati self.handle_consistency_check_complete(msg) } } + +#[cfg(test)] +mod tests { + use super::*; + use actix::{Actor, Context, Handler}; + use alloy::signers::local::PrivateKeySigner; + use e3_events::{ + hlc_factory::HlcFactory, Event, EventBus, EventBusBarrier, EventBusConfig, EventPublisher, + Proof, ProofPayload, Sequencer, StoreEventRequested, StoreEventResponse, + }; + use e3_fhe_params::BfvPreset; + use e3_utils::utility_types::ArcBytes; + use e3_zk_helpers::CiphernodesCommitteeSize; + use std::{collections::BTreeSet, time::Duration}; + + #[derive(Default)] + struct TestEventStore { + next_seq: u64, + } + + impl Actor for TestEventStore { + type Context = Context; + } + + impl Handler for TestEventStore { + type Result = (); + + fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) { + let StoreEventRequested { event, sender } = msg; + let seq = self.next_seq; + self.next_seq += 1; + sender.do_send(StoreEventResponse(event.into_sequenced(seq))); + } + } + + fn test_bus() -> BusHandle { + let event_bus = + EventBus::::new(EventBusConfig { deduplicate: true }).start(); + let store = TestEventStore::default().start(); + let sequencer = Sequencer::new(&event_bus, store.recipient()).start(); + BusHandle::new(event_bus, sequencer, HlcFactory::new()).enable("share-recovery-test") + } + + fn signed_c6(signer: &PrivateKeySigner, e3_id: &E3id, marker: u8) -> SignedProofPayload { + let proof_type = ProofType::C6ThresholdShareDecryption; + let proof = Proof::new( + proof_type.circuit_names()[0], + ArcBytes::from_bytes(&[marker, 2, 3]), + ArcBytes::from_bytes(&[4, 5, marker]), + ); + SignedProofPayload::sign( + ProofPayload { + e3_id: e3_id.clone(), + proof_type, + proof, + }, + signer, + ) + .expect("sign C6 test proof") + } + + #[actix::test] + async fn restored_committee_authorizes_c6_without_replayed_finalization_event() { + let bus = test_bus(); + let e3_id = E3id::new("0", 31_337); + let signers = [ + PrivateKeySigner::random(), + PrivateKeySigner::random(), + PrivateKeySigner::random(), + ]; + let committee = Committee::new( + signers + .iter() + .map(|signer| signer.address().to_string()) + .collect(), + ); + ShareVerificationActor::setup(&bus, HashMap::from([(e3_id.clone(), committee)])); + + // Register the observer and fence the EventBus before dispatching. No + // CommitteeFinalized event is published: this models restart after that event was + // incorporated into the durable snapshot and excluded from the replay range. + let consistency_requested = bus.wait_for(EventType::CommitmentConsistencyCheckRequested); + bus.event_bus() + .send(EventBusBarrier) + .await + .expect("event bus barrier"); + + bus.publish_without_context(ShareVerificationDispatched { + e3_id: e3_id.clone(), + kind: VerificationKind::ThresholdDecryptionProofs, + share_proofs: signers[..2] + .iter() + .enumerate() + .map(|(party_id, signer)| e3_events::PartyProofsToVerify { + sender_party_id: party_id as u64, + signed_proofs: vec![signed_c6(signer, &e3_id, party_id as u8)], + }) + .collect(), + decryption_proofs: Vec::new(), + pre_dishonest: BTreeSet::new(), + params_preset: BfvPreset::InsecureThreshold512, + committee_size: CiphernodesCommitteeSize::Minimum, + }) + .expect("publish C6 verification dispatch"); + + let event = tokio::time::timeout(Duration::from_secs(2), consistency_requested) + .await + .expect("restored committee did not authorize C6 before timeout") + .expect("wait for consistency request"); + let InterfoldEventData::CommitmentConsistencyCheckRequested(request) = event.into_data() + else { + panic!("unexpected event type") + }; + assert_eq!(request.e3_id, e3_id); + assert_eq!(request.party_proofs.len(), 2); + } +} From 00cd862c31a5ccdf3b15843e4f5e81a24d1ea865 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 21:55:19 +0500 Subject: [PATCH 50/62] fix(zk): restore C0 verification context after restart --- agent/flow-trace/04_DKG_AND_COMPUTATION.md | 2 + .../06_DEACTIVATION_AND_COMPLETION.md | 10 + .../src/ciphernode_builder.rs | 17 +- crates/zk-prover/src/actors/mod.rs | 45 ++- .../src/actors/proof_verification.rs | 287 +++++++++++++++--- crates/zk-prover/src/lib.rs | 4 +- 6 files changed, 319 insertions(+), 46 deletions(-) diff --git a/agent/flow-trace/04_DKG_AND_COMPUTATION.md b/agent/flow-trace/04_DKG_AND_COMPUTATION.md index 72bb1c05df..c1d582a5f6 100644 --- a/agent/flow-trace/04_DKG_AND_COMPUTATION.md +++ b/agent/flow-trace/04_DKG_AND_COMPUTATION.md @@ -91,6 +91,8 @@ ProofRequestActor receives EncryptionKeyPending └─ RECEIVING NODES verify C0 proof: ProofVerificationActor receives EncryptionKeyReceived (from P2P) │ + ├─ Resolves canonical party ownership plus BFV preset/committee artifact scope + │ from startup-recovered verifier context; live lifecycle events refresh both caches ├─ Recovers ECDSA signer address from signed proof ├─ Dispatches ZK verification to ZkActor: │ ZkActor runs: bb verify -k vk -p proof.data diff --git a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md index 2785ca8b5b..07322c5b32 100644 --- a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md +++ b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md @@ -221,6 +221,9 @@ On restart: │ → ShareVerificationActor loads canonical party slots from the durable │ finalized-committees repository before replay. A snapshotted │ CommitteeFinalized event is not guaranteed to appear in the replay window. +│ → ProofVerificationActor loads the same slots, plus BFV preset/threshold +│ metadata from durable CiphernodeSelector state. Snapshotted +│ CiphernodeSelected events are likewise not guaranteed to replay. │ 2. CiphernodeSelector emits persisted AggregatorChanged state before replay │ → ThresholdPlaintextAggregatorExtension records this role in the E3 context │ so a plaintext buffer created later by CiphertextOutputPublished starts @@ -332,6 +335,13 @@ startup, before EventStore replay. Relying only on a `CommitteeFinalized` subscr once that event is included in a snapshot, replay starts after it and a restarted aggregator would reject every honest C6 signer as having no canonical slot. +The global `ProofVerificationActor` has the same party-slot requirement for C0 and additionally +needs the request's BFV preset and threshold-derived committee size to choose circuit artifacts and +recompute the advertised public-key commitment. Builder startup seeds those caches from the durable +finalized-committee repository and `CiphernodeSelectorState.e3_cache` before replay. Live +`CommitteeFinalized` / `CiphernodeSelected` events remain authoritative refreshes, while +`E3RequestComplete` removes both caches. + --- ## Rust-Side: E3 Lifecycle Coordinator (durable stage tracking) diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index a9d699cd9a..5923008519 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -44,7 +44,7 @@ use e3_sortition::{ }; use e3_sync::{preflight_schema_version, sync}; use e3_utils::SharedRng; -use e3_zk_prover::{setup_zk_actors, ZkBackend}; +use e3_zk_prover::{setup_zk_actors, ZkActorRecovery, ZkBackend}; use libp2p::PeerId; use std::time::Duration; use std::{collections::HashMap, path::PathBuf, sync::Arc}; @@ -776,12 +776,19 @@ impl CiphernodeBuilder { accusation_vote_validity_by_chain: &HashMap, ) -> Result { let mut e3_builder = E3Router::builder(bus, store.clone()); - let persisted_committees = store - .repositories() + let repositories = store.repositories(); + let persisted_committees = repositories .finalized_committees() .read() .await? .unwrap_or_default(); + let persisted_e3_metadata = repositories + .ciphernode_selector() + .read() + .await? + .map(|state| state.e3_cache) + .unwrap_or_default(); + let zk_recovery = ZkActorRecovery::new(persisted_committees, persisted_e3_metadata); // ── Threshold keyshare + ZK actors ── if let Some(KeyshareKind::Threshold) = self.keyshare { @@ -803,7 +810,7 @@ impl CiphernodeBuilder { backend, _signer, dkg_fold_verifier_by_chain.clone(), - persisted_committees.clone(), + zk_recovery.clone(), ); } @@ -828,7 +835,7 @@ impl CiphernodeBuilder { backend, signer, dkg_fold_verifier_by_chain.clone(), - persisted_committees, + zk_recovery, ); } } diff --git a/crates/zk-prover/src/actors/mod.rs b/crates/zk-prover/src/actors/mod.rs index fe0562d90a..19f4901436 100644 --- a/crates/zk-prover/src/actors/mod.rs +++ b/crates/zk-prover/src/actors/mod.rs @@ -21,7 +21,7 @@ //! ## Usage //! //! ```rust,ignore -//! use e3_zk_prover::{ZkBackend, setup_zk_actors}; +//! use e3_zk_prover::{ZkActorRecovery, ZkBackend, setup_zk_actors}; //! use e3_events::BusHandle; //! use alloy::signers::local::PrivateKeySigner; //! use std::collections::HashMap; @@ -31,7 +31,13 @@ //! let signer = PrivateKeySigner::random(); //! //! // Setup all actors with proper separation of concerns -//! setup_zk_actors(&bus, &backend, signer, HashMap::new(), HashMap::new()); +//! setup_zk_actors( +//! &bus, +//! &backend, +//! signer, +//! HashMap::new(), +//! ZkActorRecovery::default(), +//! ); //! ``` pub mod accusation_manager; @@ -59,10 +65,33 @@ use actix::{Actor, Addr}; use alloy::primitives::Address; use alloy::signers::local::PrivateKeySigner; use e3_events::{BusHandle, Committee, E3id}; +use e3_request::E3Meta; use std::collections::HashMap; use crate::ZkBackend; +/// Durable inputs needed by global proof-verification actors before EventStore replay begins. +/// +/// Both maps are projections of canonical protocol events. They are startup seeds, not separate +/// authorities: live or replayed lifecycle events continue to update the actor caches. +#[derive(Clone, Debug, Default)] +pub struct ZkActorRecovery { + finalized_committees: HashMap, + e3_metadata: HashMap, +} + +impl ZkActorRecovery { + pub fn new( + finalized_committees: HashMap, + e3_metadata: HashMap, + ) -> Self { + Self { + finalized_committees, + e3_metadata, + } + } +} + /// Setup all ZK-related actors with proper separation of concerns. /// /// Requires a `ZkBackend` for proof generation/verification and a @@ -70,19 +99,25 @@ use crate::ZkBackend; /// `dkg_fold_attestation_verifiers_by_chain` maps each enabled chain's id to /// `CiphernodeRegistry.dkgFoldAttestationVerifier()` (EIP-712 `verifyingContract` /// for fold attestations). Fetched at node startup when proof aggregation is enabled. +/// `recovery` seeds global verifier context before EventStore replay. pub fn setup_zk_actors( bus: &BusHandle, backend: &ZkBackend, signer: PrivateKeySigner, dkg_fold_attestation_verifiers_by_chain: HashMap>, - persisted_committees: HashMap, + recovery: ZkActorRecovery, ) -> ZkActors { + let ZkActorRecovery { + finalized_committees, + e3_metadata, + } = recovery; let zk_actor = ZkActor::new(backend).start(); let verifier = zk_actor.clone().recipient(); let proof_request = ProofRequestActor::setup(bus, signer.clone()); - let proof_verification = ProofVerificationActor::setup(bus, verifier); - let share_verification = ShareVerificationActor::setup(bus, persisted_committees); + let proof_verification = + ProofVerificationActor::setup(bus, verifier, finalized_committees.clone(), e3_metadata); + let share_verification = ShareVerificationActor::setup(bus, finalized_committees); let node_proof_aggregator = NodeProofAggregator::setup(bus, signer, dkg_fold_attestation_verifiers_by_chain); diff --git a/crates/zk-prover/src/actors/proof_verification.rs b/crates/zk-prover/src/actors/proof_verification.rs index f45cdf2bfd..060268e58d 100644 --- a/crates/zk-prover/src/actors/proof_verification.rs +++ b/crates/zk-prover/src/actors/proof_verification.rs @@ -15,12 +15,13 @@ use actix::{Actor, Addr, AsyncContext, Context, Handler, Message, Recipient}; use alloy::primitives::{keccak256, Address, Bytes}; use alloy::sol_types::SolValue; use e3_events::{ - BusHandle, E3id, EncryptionKey, EncryptionKeyCreated, EncryptionKeyReceived, EventContext, - EventPublisher, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, Proof, - ProofType, ProofVerificationFailed, ProofVerificationPassed, Sequenced, SignedProofFailed, - SignedProofPayload, TypedEvent, + BusHandle, Committee, E3id, EncryptionKey, EncryptionKeyCreated, EncryptionKeyReceived, + EventContext, EventPublisher, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, + Proof, ProofType, ProofVerificationFailed, ProofVerificationPassed, Sequenced, + SignedProofFailed, SignedProofPayload, TypedEvent, }; use e3_fhe_params::BfvPreset; +use e3_request::E3Meta; use e3_utils::NotifySync; use e3_zk_helpers::{compute_dkg_pk_commitment_from_public_key_bytes, CiphernodesCommitteeSize}; use tracing::{error, info, warn}; @@ -64,21 +65,40 @@ pub struct ProofVerificationActor { } impl ProofVerificationActor { - pub fn new(bus: &BusHandle, verifier: Recipient>) -> Self { - Self { + pub fn new( + bus: &BusHandle, + verifier: Recipient>, + persisted_committees: HashMap, + persisted_e3_metadata: HashMap, + ) -> Self { + let mut actor = Self { bus: bus.clone(), verifier, pending: HashMap::new(), presets: HashMap::new(), committees: HashMap::new(), + }; + for (e3_id, meta) in persisted_e3_metadata { + actor.store_preset( + e3_id, + meta.params_preset, + meta.threshold_m, + meta.threshold_n, + ); } + for (e3_id, committee) in persisted_committees { + actor.store_committee(e3_id, committee.members()); + } + actor } pub fn setup( bus: &BusHandle, verifier: Recipient>, + persisted_committees: HashMap, + persisted_e3_metadata: HashMap, ) -> Addr { - let addr = Self::new(bus, verifier).start(); + let addr = Self::new(bus, verifier, persisted_committees, persisted_e3_metadata).start(); bus.subscribe(EventType::CiphernodeSelected, addr.clone().into()); bus.subscribe(EventType::CommitteeFinalized, addr.clone().into()); bus.subscribe(EventType::EncryptionKeyReceived, addr.clone().into()); @@ -86,6 +106,48 @@ impl ProofVerificationActor { addr } + fn store_preset( + &mut self, + e3_id: E3id, + preset: BfvPreset, + threshold_m: usize, + threshold_n: usize, + ) { + match CiphernodesCommitteeSize::from_threshold(threshold_m, threshold_n) { + Ok(committee) => { + self.presets.insert(e3_id, (preset, committee)); + } + Err(error) => { + error!( + %e3_id, + threshold_m, + threshold_n, + %error, + "ProofVerificationActor: unrecognised committee; C0 keys will be rejected" + ); + } + } + } + + fn store_committee(&mut self, e3_id: E3id, members: &[String]) { + match members + .iter() + .map(|node| node.parse()) + .collect::, _>>() + { + Ok(committee) => { + self.committees.insert(e3_id, committee); + } + Err(error) => { + error!( + %e3_id, + %error, + "Finalized committee contains an invalid address; C0 keys will be rejected" + ); + } + } + } + fn handle_encryption_key_received( &mut self, msg: TypedEvent, @@ -219,38 +281,18 @@ impl Handler for ProofVerificationActor { let (msg, ec) = msg.into_components(); match msg { InterfoldEventData::CiphernodeSelected(data) => { - match CiphernodesCommitteeSize::from_threshold(data.threshold_m, data.threshold_n) { - Ok(committee) => { - self.presets - .insert(data.e3_id.clone(), (data.params_preset, committee)); - } - Err(e) => { - error!( - "ProofVerificationActor: unrecognised committee for E3 {} \ - (threshold_m={}, threshold_n={}): {e} — skipping preset registration, \ - proof verification will be rejected if a key arrives", - data.e3_id, data.threshold_m, data.threshold_n - ); - } - } + self.store_preset( + data.e3_id, + data.params_preset, + data.threshold_m, + data.threshold_n, + ); } InterfoldEventData::CommitteeFinalized(mut data) => { // The EVM decoder already emits canonical address order, but sorting again keeps // this trust boundary correct for replayed/test-produced events as well. data.sort_by_score(); - let parsed: Result, _> = - data.committee.iter().map(|node| node.parse()).collect(); - match parsed { - Ok(committee) => { - self.committees.insert(data.e3_id, committee); - } - Err(err) => { - error!( - e3_id = %data.e3_id, - "Finalized committee contains an invalid address: {err}; C0 keys will be rejected" - ); - } - } + self.store_committee(data.e3_id, &data.committee); } InterfoldEventData::EncryptionKeyReceived(data) => { self.notify_sync(ctx, TypedEvent::new(data, ec)) @@ -397,3 +439,180 @@ impl Handler> for ProofVerificationActor { } } } + +#[cfg(test)] +mod tests { + use super::*; + use actix::{Actor, Context, Handler}; + use alloy::signers::local::PrivateKeySigner; + use e3_events::{ + hlc_factory::HlcFactory, EventBus, EventBusBarrier, EventBusConfig, EventPublisher, + ProofPayload, Seed, Sequencer, StoreEventRequested, StoreEventResponse, + }; + use e3_fhe_params::build_pair_for_preset; + use e3_utils::utility_types::ArcBytes; + use fhe::bfv::{PublicKey, SecretKey}; + use fhe_traits::Serialize; + use tokio::sync::mpsc; + + #[derive(Default)] + struct TestEventStore { + next_seq: u64, + } + + impl Actor for TestEventStore { + type Context = Context; + } + + impl Handler for TestEventStore { + type Result = (); + + fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) { + let StoreEventRequested { event, sender } = msg; + let seq = self.next_seq; + self.next_seq += 1; + sender.do_send(StoreEventResponse(event.into_sequenced(seq))); + } + } + + #[derive(Debug, PartialEq, Eq)] + struct ObservedVerification { + e3_id: E3id, + party_id: u64, + artifacts_dir: String, + } + + struct VerificationRecorder { + sender: mpsc::UnboundedSender, + } + + impl Actor for VerificationRecorder { + type Context = Context; + } + + impl Handler> for VerificationRecorder { + type Result = (); + + fn handle(&mut self, request: TypedEvent, _: &mut Self::Context) { + let (request, _) = request.into_components(); + let _ = self.sender.send(ObservedVerification { + e3_id: request.e3_id, + party_id: request.key.party_id, + artifacts_dir: request.artifacts_dir, + }); + } + } + + fn test_bus() -> BusHandle { + let event_bus = + EventBus::::new(EventBusConfig { deduplicate: true }).start(); + let store = TestEventStore::default().start(); + let sequencer = Sequencer::new(&event_bus, store.recipient()).start(); + BusHandle::new(event_bus, sequencer, HlcFactory::new()).enable("c0-recovery-test") + } + + fn signed_c0_key( + signer: &PrivateKeySigner, + party_id: u64, + e3_id: &E3id, + preset: BfvPreset, + ) -> Arc { + let (_, dkg_params) = build_pair_for_preset(preset).expect("build BFV test parameters"); + let mut rng = rand::rng(); + let secret_key = SecretKey::random(&dkg_params, &mut rng); + let public_key = PublicKey::new(&secret_key, &mut rng); + let public_key_bytes = public_key.to_bytes(); + let commitment = compute_dkg_pk_commitment_from_public_key_bytes(&public_key_bytes, preset) + .expect("compute C0 key commitment"); + let proof = Proof::new( + ProofType::C0PkBfv.circuit_names()[0], + ArcBytes::from_bytes(&[1, 2, 3]), + ArcBytes::from_bytes(&commitment), + ); + let signed = SignedProofPayload::sign( + ProofPayload { + e3_id: e3_id.clone(), + proof_type: ProofType::C0PkBfv, + proof: proof.clone(), + }, + signer, + ) + .expect("sign C0 test proof"); + + Arc::new( + EncryptionKey::new(party_id, ArcBytes::from_bytes(&public_key_bytes)) + .with_proof(proof) + .with_signed_payload(signed), + ) + } + + #[actix::test] + async fn restored_context_dispatches_c0_without_replayed_lifecycle_events() { + let bus = test_bus(); + let e3_id = E3id::new("7", 31_337); + let preset = BfvPreset::InsecureThreshold512; + let signer = PrivateKeySigner::random(); + let mut committee_members = vec![ + signer.address().to_string(), + PrivateKeySigner::random().address().to_string(), + PrivateKeySigner::random().address().to_string(), + ]; + committee_members.sort_by_key(|member| member.to_lowercase()); + let party_id = committee_members + .iter() + .position(|member| member.eq_ignore_ascii_case(&signer.address().to_string())) + .expect("signer belongs to restored committee") as u64; + + let persisted_committees = + HashMap::from([(e3_id.clone(), Committee::new(committee_members))]); + let persisted_e3_metadata = HashMap::from([( + e3_id.clone(), + E3Meta { + threshold_m: 1, + threshold_n: 3, + seed: Seed([0; 32]), + params_preset: preset, + params: ArcBytes::default(), + error_size: ArcBytes::default(), + proof_aggregation_enabled: false, + }, + )]); + let (observed_tx, mut observed_rx) = mpsc::unbounded_channel(); + let verifier = VerificationRecorder { + sender: observed_tx, + } + .start(); + ProofVerificationActor::setup( + &bus, + verifier.recipient(), + persisted_committees, + persisted_e3_metadata, + ); + + // No CiphernodeSelected or CommitteeFinalized event is published: both are already + // represented by durable snapshots and may be outside the post-snapshot replay range. + bus.event_bus() + .send(EventBusBarrier) + .await + .expect("event bus barrier"); + bus.publish_without_context(EncryptionKeyReceived { + e3_id: e3_id.clone(), + key: signed_c0_key(&signer, party_id, &e3_id, preset), + }) + .expect("publish external C0 key"); + + let observed = tokio::time::timeout(std::time::Duration::from_secs(2), observed_rx.recv()) + .await + .expect("restored C0 context did not dispatch verification before timeout") + .expect("verification recorder stopped"); + assert_eq!( + observed, + ObservedVerification { + e3_id, + party_id, + artifacts_dir: preset + .artifacts_dir_for_committee(CiphernodesCommitteeSize::Minimum.as_str()), + } + ); + } +} diff --git a/crates/zk-prover/src/lib.rs b/crates/zk-prover/src/lib.rs index eec058f3c4..08948a970d 100644 --- a/crates/zk-prover/src/lib.rs +++ b/crates/zk-prover/src/lib.rs @@ -20,8 +20,8 @@ mod witness; pub use actors::commitment_links::default_links; pub use actors::{ setup_zk_actors, CommitmentConsistencyCheckerExtension, ProofRequestActor, - ProofVerificationActor, ShareVerificationActor, ZkActors, ZkVerificationRequest, - ZkVerificationResponse, + ProofVerificationActor, ShareVerificationActor, ZkActorRecovery, ZkActors, + ZkVerificationRequest, ZkVerificationResponse, }; pub use backend::{SetupStatus, ZkBackend}; From d74cd2d86ac44a1e839e2a85ba5c1ffe367355d2 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 13 Jul 2026 19:24:31 +0500 Subject: [PATCH 51/62] fix(sync): make startup replay end-to-end backpressured --- Cargo.lock | 4 + crates/events/src/eventbus.rs | 244 +++++++++++++++---- crates/events/src/eventstore.rs | 121 +++++++++- crates/sync/Cargo.toml | 2 + crates/sync/src/actors/sync.rs | 156 +++++++----- crates/sync/src/lib.rs | 1 + crates/sync/src/replay_spool.rs | 406 ++++++++++++++++++++++++++++++++ 7 files changed, 814 insertions(+), 120 deletions(-) create mode 100644 crates/sync/src/replay_spool.rs diff --git a/Cargo.lock b/Cargo.lock index f74a19b708..a1bd56c50a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3793,12 +3793,14 @@ version = "0.2.3" dependencies = [ "actix", "anyhow", + "bincode 1.3.3", "e3-ciphernode-builder", "e3-config", "e3-data", "e3-events", "e3-test-helpers", "e3-utils", + "tempfile", "tokio", "tracing", ] @@ -7076,6 +7078,8 @@ dependencies = [ "rand 0.9.2", "serde_json", "thiserror 2.0.18", + "tokio", + "tokio-stream", "tracing", ] diff --git a/crates/events/src/eventbus.rs b/crates/events/src/eventbus.rs index a1b3443c51..0e43bac7d5 100644 --- a/crates/events/src/eventbus.rs +++ b/crates/events/src/eventbus.rs @@ -8,6 +8,7 @@ use crate::traits::{ErrorEvent, Event}; use crate::{EventBusBarrier, EventType}; use actix::prelude::*; use e3_utils::{colorize, Color, MAILBOX_LIMIT, MAILBOX_LIMIT_LARGE}; +use futures_util::future::join_all; use std::collections::{HashMap, HashSet, VecDeque}; use std::fmt; use std::hash::Hash; @@ -36,6 +37,11 @@ impl Default for EventBusConfig { /// evicted in first-observed order and a later occurrence is accepted again. const DEFAULT_DEDUP_CAPACITY: usize = 250_000; +/// Actor handlers are expected to hand long-running work to child futures. A +/// full mailbox that cannot accept one event within this window is unhealthy; +/// replay fails closed instead of hanging startup forever. +const FANOUT_ACCEPT_TIMEOUT: Duration = Duration::from_secs(30); + /// A bounded, exact FIFO set. /// /// `HashSet` uses equality to resolve hash collisions, so two distinct event IDs @@ -162,8 +168,73 @@ impl EventBus { fn is_duplicate(&self, event: &E) -> bool { self.config.deduplicate && self.ids.contains(&event.event_id()) } + + fn live_listeners_for(&mut self, event_type: &str) -> Vec> { + let mut listeners = Vec::new(); + if let Some(all) = self.listeners.get_mut("*") { + all.retain(Recipient::connected); + listeners.extend(all.iter().cloned()); + } + if event_type != "*" { + if let Some(specific) = self.listeners.get_mut(event_type) { + specific.retain(Recipient::connected); + listeners.extend(specific.iter().cloned()); + } + } + listeners + } + + fn prepare_fanout(&mut self, event: &E) -> Option<(String, Vec>)> { + if self.is_duplicate(event) { + return None; + } + + let event_type = event.event_type(); + let listeners = self.live_listeners_for(&event_type); + if event_type == EventType::Shutdown.as_str() { + tracing::info!( + listeners = listeners.len(), + "Acknowledging Shutdown across EventBus subscribers" + ); + } + + tracing::info!("{} {}", colorize(">>>", Color::Yellow), event); + self.track(event); + Some((event_type, listeners)) + } } +async fn fanout( + event: E, + event_type: String, + listeners: Vec>, +) -> anyhow::Result<()> { + let deliveries = listeners.into_iter().map(|listener| { + let event = event.clone(); + async move { tokio::time::timeout(FANOUT_ACCEPT_TIMEOUT, listener.send(event)).await } + }); + + for delivery in join_all(deliveries).await { + match delivery { + Ok(Ok(())) => {} + Ok(Err(error)) => anyhow::bail!( + "EventBus subscriber was unavailable while accepting {event_type}: {error}" + ), + Err(_) => anyhow::bail!( + "EventBus subscriber did not accept {event_type} within {:?}", + FANOUT_ACCEPT_TIMEOUT + ), + } + } + Ok(()) +} + +/// Replay-only acknowledged delivery. Unlike the ordinary event message, this +/// returns mailbox saturation to the sync caller so startup can fail closed. +#[derive(Message)] +#[rtype(result = "anyhow::Result<()>")] +pub struct EventBusFanout(pub E); + impl Default for EventBus { fn default() -> Self { Self { @@ -178,62 +249,52 @@ impl Handler for EventBus { type Result = (); fn handle(&mut self, event: E, ctx: &mut Context) { - if self.is_duplicate(&event) { + let Some((event_type, listeners)) = self.prepare_fanout(&event) else { return; - } - - let event_type = event.event_type(); - if event_type == EventType::Shutdown.as_str() { - let listeners = self - .listeners - .get("*") - .into_iter() - .flatten() - .chain(self.listeners.get(&event_type).into_iter().flatten()) - .cloned() - .collect::>(); - - tracing::info!( - listeners = listeners.len(), - "Acknowledging Shutdown across EventBus subscribers" - ); - tracing::info!("{} {}", colorize(">>>", Color::Yellow), event); - self.track(&event); - - ctx.wait( - async move { - for listener in listeners { - if let Err(error) = listener.send(event.clone()).await { - tracing::warn!(%error, "Shutdown subscriber was already unavailable"); - } + }; + + // `Recipient::do_send` bypasses mailbox capacity. During startup replay that turns a + // bounded EventBus into an unbounded multiplier: every replayed event is cloned into each + // subscriber even when its mailbox is full. Await subscriber capacity concurrently and + // pause this actor so at most one subsequent event can be queued while downstream actors + // catch up. A wedged subscriber is bounded by FANOUT_ACCEPT_TIMEOUT. + // This also makes EventBusBarrier an acknowledgement of completed fanout rather than mere + // enqueueing. + ctx.wait( + async move { fanout(event, event_type.clone(), listeners).await } + .into_actor(self) + .map(|result, _, _| { + if let Err(error) = result { + tracing::error!(%error, "EventBus fanout did not complete"); } - } - .into_actor(self), - ); - return; - } + }), + ); + } +} - if let Some(listeners) = self.listeners.get("*") { - for listener in listeners { - listener.do_send(event.clone()); - } - } +impl Handler> for EventBus { + type Result = AtomicResponse>; - if let Some(listeners) = self.listeners.get(&event_type) { - for listener in listeners { - listener.do_send(event.clone()); + fn handle(&mut self, msg: EventBusFanout, _: &mut Context) -> Self::Result { + let event = msg.0; + let prepared = self.prepare_fanout(&event); + AtomicResponse::new(Box::pin( + async move { + if let Some((event_type, listeners)) = prepared { + fanout(event, event_type, listeners).await + } else { + Ok(()) + } } - } - - tracing::info!("{} {}", colorize(">>>", Color::Yellow), event); - self.track(&event); + .into_actor(self), + )) } } impl Handler for EventBus { type Result = (); - fn handle(&mut self, _: EventBusBarrier, _: &mut Context) {} + fn handle(&mut self, _: EventBusBarrier, _: &mut Context) -> Self::Result {} } ////////////////////////////////////////////////////////////////////////////// @@ -527,7 +588,7 @@ mod tests { atomic::{AtomicBool, Ordering}, Arc, }; - use tokio::sync::Notify; + use tokio::sync::{mpsc, Notify}; #[derive(Clone, Debug, Eq, PartialEq)] struct CollidingId(u64); @@ -608,16 +669,16 @@ mod tests { } } - struct BlockingShutdownSubscriber { + struct BlockingSubscriber { gate: Arc, completed: Arc, } - impl Actor for BlockingShutdownSubscriber { + impl Actor for BlockingSubscriber { type Context = actix::Context; } - impl Handler for BlockingShutdownSubscriber { + impl Handler for BlockingSubscriber { type Result = ResponseFuture<()>; fn handle(&mut self, _: CollidingEvent, _: &mut Self::Context) -> Self::Result { @@ -630,6 +691,20 @@ mod tests { } } + struct NotifyingSubscriber(mpsc::UnboundedSender); + + impl Actor for NotifyingSubscriber { + type Context = actix::Context; + } + + impl Handler for NotifyingSubscriber { + type Result = (); + + fn handle(&mut self, event: CollidingEvent, _: &mut Self::Context) { + let _ = self.0.send(event.id.0); + } + } + #[actix::test] async fn distinct_ids_with_identical_hashes_are_both_delivered() -> anyhow::Result<()> { let bus = EventBus::with_dedup_capacity(EventBusConfig::default(), 4).start(); @@ -697,7 +772,7 @@ mod tests { let bus = EventBus::with_dedup_capacity(EventBusConfig::default(), 2).start(); let gate = Arc::new(Notify::new()); let completed = Arc::new(AtomicBool::new(false)); - let subscriber = BlockingShutdownSubscriber { + let subscriber = BlockingSubscriber { gate: Arc::clone(&gate), completed: Arc::clone(&completed), } @@ -720,4 +795,73 @@ mod tests { assert!(completed.load(Ordering::SeqCst)); Ok(()) } + + #[actix::test] + async fn ordinary_fanout_pauses_event_bus_until_subscriber_acknowledges() -> anyhow::Result<()> + { + let bus = EventBus::with_dedup_capacity(EventBusConfig::default(), 2).start(); + let gate = Arc::new(Notify::new()); + let completed = Arc::new(AtomicBool::new(false)); + let subscriber = BlockingSubscriber { + gate: Arc::clone(&gate), + completed: Arc::clone(&completed), + } + .start(); + + bus.send(Subscribe::new("CollidingEvent", subscriber.recipient())) + .await?; + bus.send(CollidingEvent::new(9)).await?; + + let mut fence = Box::pin(bus.send(EventBusBarrier)); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut fence) + .await + .is_err(), + "EventBus fence completed before the ordinary subscriber" + ); + + gate.notify_one(); + fence.await?; + assert!(completed.load(Ordering::SeqCst)); + Ok(()) + } + + #[actix::test] + async fn acknowledged_fanout_does_not_head_of_line_block_other_subscribers( + ) -> anyhow::Result<()> { + let bus = EventBus::with_dedup_capacity(EventBusConfig::default(), 2).start(); + let gate = Arc::new(Notify::new()); + let completed = Arc::new(AtomicBool::new(false)); + let blocked = BlockingSubscriber { + gate: Arc::clone(&gate), + completed, + } + .start(); + let (tx, mut rx) = mpsc::unbounded_channel(); + let immediate = NotifyingSubscriber(tx).start(); + + bus.send(Subscribe::new("CollidingEvent", blocked.recipient())) + .await?; + bus.send(Subscribe::new("CollidingEvent", immediate.recipient())) + .await?; + + let mut fanout = Box::pin(bus.send(EventBusFanout(CollidingEvent::new(9)))); + let received = tokio::time::timeout(Duration::from_millis(100), rx.recv()) + .await? + .expect("healthy subscriber stopped"); + assert_eq!( + received, 9, + "the healthy subscriber should receive concurrently" + ); + assert!( + tokio::time::timeout(Duration::from_millis(20), &mut fanout) + .await + .is_err(), + "acknowledged fanout completed while one subscriber was still blocked" + ); + + gate.notify_one(); + fanout.await??; + Ok(()) + } } diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 8e65b03ca3..25f80b5297 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -9,7 +9,7 @@ use crate::{ Event, EventContextAccessors, EventLog, EventStoreFilter, EventStoreQueryBy, EventStoreQueryResponse, InterfoldEvent, Seq, SequenceIndex, Sequenced, Ts, Unsequenced, }; -use actix::{Actor, Handler}; +use actix::{Actor, AsyncContext, Handler, Recipient, WrapFuture}; use anyhow::{bail, Result}; use tracing::{error, warn}; @@ -264,41 +264,64 @@ impl Handler for EventStore Handler> for EventStore { type Result = (); - fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EventStoreQueryBy, ctx: &mut Self::Context) -> Self::Result { let query = msg.query(); let id = msg.id(); let limit = msg.limit(); let filter = msg.filter().cloned(); let sender = msg.sender(); - match self.query_by_ts(query, filter, limit) { - Ok(evts) => { - if let Err(e) = sender.try_send(EventStoreQueryResponse::new(id, evts)) { - error!("{e}"); + let response = self + .query_by_ts(query, filter, limit) + .map(|events| EventStoreQueryResponse::new(id, events)); + ctx.wait( + async move { + match response { + Ok(response) => { + if let Err(error) = deliver_query_response(sender, response).await { + error!(%error, "Event-store query recipient closed"); + } + } + Err(error) => error!(%error, "Event-store timestamp query failed"), } } - Err(e) => error!("{e}"), - } + .into_actor(self), + ); } } impl Handler> for EventStore { type Result = (); - fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) -> Self::Result { + fn handle(&mut self, msg: EventStoreQueryBy, ctx: &mut Self::Context) -> Self::Result { let id = msg.id(); let query = msg.query(); let limit = msg.limit(); let filter = msg.filter().cloned(); let sender = msg.sender(); let evts = self.query_by_seq(query, filter, limit); - if let Err(e) = sender.try_send(EventStoreQueryResponse::new(id, evts)) { - error!("{e}"); - } + let response = EventStoreQueryResponse::new(id, evts); + ctx.wait( + async move { + if let Err(error) = deliver_query_response(sender, response).await { + error!(%error, "Event-store query recipient closed"); + } + } + .into_actor(self), + ); } } +async fn deliver_query_response( + sender: Recipient, + response: EventStoreQueryResponse, +) -> std::result::Result<(), actix::MailboxError> { + sender.send(response).await +} + #[cfg(test)] mod tests { - use crate::{EventConstructorWithTimestamp, EventContextSeq, EventSource, TestEvent}; + use crate::{ + CorrelationId, EventConstructorWithTimestamp, EventContextSeq, EventSource, TestEvent, + }; use super::*; use anyhow::Result; @@ -535,6 +558,78 @@ mod tests { assert_eq!(flushes.load(Ordering::SeqCst), 1); Ok(()) } + + #[actix::test] + async fn query_response_waits_for_a_full_recipient_mailbox() { + use tokio::sync::{mpsc, Notify}; + + struct SaturatedResponseSink { + block_first: bool, + gate: Arc, + received: mpsc::UnboundedSender<()>, + } + + impl Actor for SaturatedResponseSink { + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(1); + } + } + + impl Handler for SaturatedResponseSink { + type Result = (); + + fn handle( + &mut self, + _: EventStoreQueryResponse, + ctx: &mut Self::Context, + ) -> Self::Result { + self.received.send(()).unwrap(); + if self.block_first { + self.block_first = false; + let gate = Arc::clone(&self.gate); + ctx.wait(async move { gate.notified().await }.into_actor(self)); + } + } + } + + let gate = Arc::new(Notify::new()); + let (received_tx, mut received_rx) = mpsc::unbounded_channel(); + let sink = SaturatedResponseSink { + block_first: true, + gate: Arc::clone(&gate), + received: received_tx, + } + .start(); + let recipient = sink.recipient(); + + recipient.do_send(EventStoreQueryResponse::new(CorrelationId::new(), vec![])); + received_rx.recv().await.unwrap(); + recipient + .try_send(EventStoreQueryResponse::new(CorrelationId::new(), vec![])) + .unwrap(); + + let delivery = tokio::spawn(deliver_query_response( + recipient, + EventStoreQueryResponse::new(CorrelationId::new(), vec![]), + )); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + assert!( + !delivery.is_finished(), + "delivery must wait instead of dropping a full-mailbox response" + ); + + gate.notify_one(); + tokio::time::timeout(std::time::Duration::from_secs(1), delivery) + .await + .expect("response delivery should resume after mailbox capacity is available") + .unwrap() + .unwrap(); + received_rx.recv().await.unwrap(); + received_rx.recv().await.unwrap(); + } + #[test] fn startup_index_reconciliation_reads_log_in_bounded_pages() { let event_count = INDEX_RECONCILE_PAGE_SIZE * 2 + 2; diff --git a/crates/sync/Cargo.toml b/crates/sync/Cargo.toml index e4136d42a5..94364435ae 100644 --- a/crates/sync/Cargo.toml +++ b/crates/sync/Cargo.toml @@ -9,12 +9,14 @@ repository = "https://github.com/theinterfold/interfold/crates/sync" [dependencies] actix.workspace = true anyhow.workspace = true +bincode.workspace = true e3-events.workspace = true e3-data.workspace = true e3-config.workspace = true e3-utils.workspace = true tokio.workspace = true tracing.workspace = true +tempfile.workspace = true [dev-dependencies] e3-events = { workspace = true, features = ["test-helpers"] } diff --git a/crates/sync/src/actors/sync.rs b/crates/sync/src/actors/sync.rs index db9770f06d..fe9d8997b0 100644 --- a/crates/sync/src/actors/sync.rs +++ b/crates/sync/src/actors/sync.rs @@ -4,25 +4,33 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +#[cfg(test)] +use crate::domain::ReplayDecision; use crate::domain::{ - decide_schema_version, CollectOutcome, HistoricalEvmCollector, ReplayDecision, - SchemaVersionDecision, SnapshotMeta, SyncPlanner, SCHEMA_VERSION, + decide_schema_version, CollectOutcome, HistoricalEvmCollector, SchemaVersionDecision, + SnapshotMeta, SyncPlanner, SCHEMA_VERSION, }; +use crate::replay_spool::ReplaySpool; use crate::SyncRepositoryFactory; use actix::{Message, Recipient}; use anyhow::{bail, Context, Result}; use e3_data::Repositories; use e3_events::{ - AggregateConfig, BusHandle, CorrelationId, EffectsEnabled, Event, EventContextAccessors, - EventPublisher, EventStoreQueryBy, EventStoreQueryResponse, EventSubscriber, EventType, - EvmEventConfig, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, HistoricalNetSyncStart, - InterfoldEvent, InterfoldEventData, SeqAgg, StoreKeys, SyncEnded, Unsequenced, + AggregateConfig, BusHandle, CorrelationId, EffectsEnabled, Event, EventPublisher, + EventStoreQueryBy, EventStoreQueryResponse, EventSubscriber, EventType, EvmEventConfig, + HistoricalEvmEventsReceived, HistoricalEvmSyncStart, HistoricalNetSyncStart, InterfoldEvent, + InterfoldEventData, SeqAgg, StoreKeys, SyncEnded, Unsequenced, }; +#[cfg(test)] +use e3_events::{EventBusBarrier, EventBusFanout, EventContextAccessors}; use e3_utils::actix::channel as actix_toolbox; use std::time::Duration; use tokio::sync::mpsc::Receiver; use tracing::info; +#[cfg(test)] +const REPLAY_PROGRESS_INTERVAL: usize = 10_000; + pub async fn sync( bus: &BusHandle, default_config: &EvmEventConfig, @@ -59,18 +67,11 @@ pub async fn sync( let evm_config = snapshot.to_evm_config(); let snapshot_net_config = snapshot.to_net_config(); - // 3. Load EventStore events since the sequence number found in the snapshot into memory. - info!("Loading EventStore events..."); - let (addr, rx) = actix_toolbox::oneshot::(); - eventstore.try_send(EventStoreQueryBy::::new( - CorrelationId::new(), - snapshot.to_sequence_map(), - addr, - ))?; - let mut events = rx.await?.into_events(); - info!("{} EventStore events loaded.", events.len()); - seed_clock_from_replay(bus, &events)?; - sort_replay_events(&mut events); + // 3. Page post-snapshot EventStore history into sorted temporary runs. This preserves the + // global HLC replay order without retaining the complete backlog in memory. + info!("Loading EventStore replay pages..."); + let replay_spool = ReplaySpool::load(eventstore, snapshot.to_sequence_map()).await?; + info!("{} EventStore events spooled.", replay_spool.total_events()); info!("Replaying events to actors..."); // 4. Replay the EventStore events to all listeners (except effects). @@ -81,13 +82,8 @@ pub async fn sync( // EventId (payload hash) as the one we publish later, causing the later event to be // silently dropped. This is critical for SyncEnded, if the EvmChainGateway never // receives it, the gateway stays in BufferUntilLive and all live EVM events are lost. - for event in events { - if SyncPlanner::classify_replay(&event) == ReplayDecision::SkipInfrastructure { - continue; - } - bus.event_bus().try_send(event)?; - } - info!("Events replayed."); + let replayed = replay_spool.replay(bus).await?; + info!(replayed_events = replayed, "Events replayed."); // Loose ends after a crash: // @@ -188,21 +184,48 @@ async fn publish_reconciled_history( Ok(()) } -fn seed_clock_from_replay(bus: &BusHandle, events: &[InterfoldEvent]) -> Result<()> { +#[cfg(test)] +async fn replay_eventstore_events( + bus: &BusHandle, + mut events: Vec, +) -> Result { + let total_events = events.len(); + let mut replayed = 0usize; + // Snapshot metadata can lag the append-only log after a failed snapshot write. Seed from the // actual replay set before any subscriber can emit follow-up work, otherwise new local events // may receive timestamps behind durable post-snapshot history. if let Some(max_ts) = events.iter().map(EventContextAccessors::ts).max() { bus.seed_clock(max_ts)?; } - Ok(()) -} -fn sort_replay_events(events: &mut [InterfoldEvent]) { // EventStoreRouter gathers one query response per aggregate. Those actor responses can arrive // in any order, so replay must re-establish the global HLC order before stateful subscribers // observe cross-aggregate dependencies. events.sort_by_key(|event| event.ts()); + + for event in events { + if SyncPlanner::classify_replay(&event) == ReplayDecision::SkipInfrastructure { + continue; + } + // Await EventBus handling before submitting the next event. `try_send` lets this producer + // outrun the bounded mailbox and aborts startup when it fills; the awaited Actix request + // preserves replay order, yields between events, and reports a closed mailbox. + bus.event_bus().send(EventBusFanout(event)).await??; + replayed += 1; + + if replayed.is_multiple_of(REPLAY_PROGRESS_INTERVAL) { + info!( + replayed_events = replayed, + total_events, "EventStore replay progress" + ); + } + } + // The EventBus acknowledges its own handler before an awaited subscriber fanout finishes. + // A fence queued after the final replay event therefore proves the last downstream handler + // has completed before startup advances to canonical-chain reconciliation. + bus.event_bus().send(EventBusBarrier).await?; + Ok(replayed) } /// Validate or initialize the durable schema marker before runtime actors can write state. @@ -336,18 +359,19 @@ impl SnapshotLoaded { mod tests { use super::{ collect_historical_evm_events, has_schema_governed_kv_state, preflight_schema_version, - publish_reconciled_history, seed_clock_from_replay, sort_replay_events, + publish_reconciled_history, replay_eventstore_events, }; - use crate::domain::SyncPlanner; use crate::SyncRepositoryFactory; use e3_ciphernode_builder::EventSystem; use e3_data::Repositories; use e3_events::{ hlc::{Hlc, HlcTimestamp}, - EffectsEnabled, Event, EventPublisher, EventSubscriber, EventType, EvmEventConfig, - EvmEventConfigChain, GetEvents, HistoricalEvmEventsReceived, HistoricalEvmSyncStart, - InterfoldEvent, InterfoldEventData, StoreKeys, SyncEnded, TakeEvents, Unsequenced, + EffectsEnabled, Event, EventContextAccessors, EventPublisher, EventSubscriber, EventType, + EvmEventConfig, EvmEventConfigChain, GetEvents, HistoricalEvmEventsReceived, + HistoricalEvmSyncStart, InterfoldEvent, InterfoldEventData, StoreKeys, SyncEnded, + TakeEvents, Unsequenced, }; + use e3_utils::MAILBOX_LIMIT_LARGE; use std::collections::BTreeMap; fn make_historical_evm_sync_start() -> HistoricalEvmSyncStart { @@ -561,12 +585,8 @@ mod tests { .build(), ]; - for event in events { - if SyncPlanner::is_infrastructure_event(&event) { - continue; - } - bus.event_bus().try_send(event)?; - } + let replayed = replay_eventstore_events(&bus, events).await?; + assert_eq!(replayed, 2); let received = history.send(TakeEvents::new(2)).await?; @@ -601,21 +621,26 @@ mod tests { } #[actix::test] - async fn replay_seeds_clock_from_post_snapshot_log_history() -> anyhow::Result<()> { + async fn replay_backlog_larger_than_event_bus_mailbox_is_delivered() -> anyhow::Result<()> { let system = EventSystem::new().with_fresh_bus(); - let bus = system - .handle()? - .enable_with_hlc(Hlc::new(7).with_clock(|| 1_000)); - let durable = HlcTimestamp::new(5_000, 17, 99); - let events = vec![InterfoldEvent::::test_event("durable") - .id(1) - .ts(durable.to_u128()) - .seq(1) - .build()]; + let bus = system.handle()?.enable("test-large-sync-replay"); + let history = bus.history(); + let count = MAILBOX_LIMIT_LARGE * 2; + let events = (0..count) + .map(|i| { + InterfoldEvent::::test_event("replay") + .id(i as u64 + 1) + .seq(i as u64 + 1) + .build() + }) + .collect(); - seed_clock_from_replay(&bus, &events)?; + let replayed = replay_eventstore_events(&bus, events).await?; + assert_eq!(replayed, count); - assert!(HlcTimestamp::from(bus.ts()?) > durable); + let received = history.send(TakeEvents::new(count)).await?; + assert!(!received.timed_out, "all replay events should be delivered"); + assert_eq!(received.events.len(), count); Ok(()) } @@ -624,7 +649,7 @@ mod tests { let system = EventSystem::new().with_fresh_bus(); let bus = system.handle()?.enable("test-ordered-sync-replay"); let history = bus.history(); - let mut events = vec![ + let events = vec![ InterfoldEvent::::test_event("third") .id(3) .aggregate_id(3) @@ -645,10 +670,7 @@ mod tests { .build(), ]; - sort_replay_events(&mut events); - for event in events { - bus.event_bus().try_send(event)?; - } + replay_eventstore_events(&bus, events).await?; let received = history.send(TakeEvents::new(3)).await?; let timestamps = received @@ -659,6 +681,26 @@ mod tests { assert_eq!(timestamps, vec![10, 20, 30]); Ok(()) } + + #[actix::test] + async fn replay_seeds_clock_from_post_snapshot_log_history() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system + .handle()? + .enable_with_hlc(Hlc::new(7).with_clock(|| 1_000)); + let durable = HlcTimestamp::new(5_000, 17, 99); + let events = vec![InterfoldEvent::::test_event("durable") + .id(1) + .ts(durable.to_u128()) + .seq(1) + .build()]; + + replay_eventstore_events(&bus, events).await?; + + assert!(HlcTimestamp::from(bus.ts()?) > durable); + Ok(()) + } + #[actix::test] async fn startup_history_is_fenced_between_effects_and_live_mode() -> anyhow::Result<()> { let system = EventSystem::new().with_fresh_bus(); diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs index a78050b8c3..47b82243fe 100644 --- a/crates/sync/src/lib.rs +++ b/crates/sync/src/lib.rs @@ -6,6 +6,7 @@ mod actors; mod domain; +mod replay_spool; mod repo; pub use actors::*; diff --git a/crates/sync/src/replay_spool.rs b/crates/sync/src/replay_spool.rs new file mode 100644 index 0000000000..78545ece60 --- /dev/null +++ b/crates/sync/src/replay_spool.rs @@ -0,0 +1,406 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Bounded external sort for post-snapshot EventStore replay. + +use actix::Recipient; +use anyhow::{bail, Context, Result}; +use e3_events::{ + AggregateId, BusHandle, CorrelationId, EventBusBarrier, EventBusFanout, EventContextAccessors, + EventContextSeq, EventStoreQueryBy, EventStoreQueryResponse, InterfoldEvent, SeqAgg, +}; +use e3_utils::actix::channel as actix_toolbox; +use std::{ + cmp::Ordering, + collections::BinaryHeap, + fs::File, + io::{BufReader, BufWriter, Read, Write}, +}; +use tempfile::NamedTempFile; +use tracing::info; + +use crate::{ReplayDecision, SyncPlanner}; + +const REPLAY_QUERY_PAGE_SIZE: usize = 1_024; +const REPLAY_MERGE_FAN_IN: usize = 32; +const MAX_SPOOLED_EVENT_BYTES: usize = 64 * 1024 * 1024; +const REPLAY_PROGRESS_INTERVAL: usize = 10_000; + +fn first_replay_sequence(snapshot_cursor: u64) -> u64 { + snapshot_cursor.max(1) +} + +/// Sorted temporary runs plus the ordering metadata discovered while paging the +/// EventStore. Temporary files are deleted automatically on every exit path. +pub(crate) struct ReplaySpool { + runs: Vec, + total_events: usize, + max_timestamp: Option, +} + +impl ReplaySpool { + pub(crate) async fn load( + eventstore: &Recipient>, + cursors: std::collections::HashMap, + ) -> Result { + let mut ordered_cursors: Vec<_> = cursors.into_iter().collect(); + ordered_cursors.sort_by_key(|(aggregate_id, _)| *aggregate_id); + + let mut runs = Vec::new(); + let mut total_events = 0usize; + let mut max_timestamp: Option = None; + + for (aggregate_id, snapshot_cursor) in ordered_cursors { + // Event-log sequence numbers are one-based while a fresh snapshot uses zero. + // Querying zero is supported by the legacy store API, but it still returns event + // sequence one; normalize here so integrity validation has one unambiguous cursor. + let mut cursor = first_replay_sequence(snapshot_cursor); + loop { + let mut page = query_page(eventstore, aggregate_id, cursor).await?; + if page.is_empty() { + break; + } + if page.len() > REPLAY_QUERY_PAGE_SIZE { + bail!( + "EventStore returned {} replay events for aggregate {}, exceeding page limit {}", + page.len(), + aggregate_id, + REPLAY_QUERY_PAGE_SIZE + ); + } + + let mut expected_sequence = cursor; + for event in &page { + if event.aggregate_id() != aggregate_id { + bail!( + "EventStore returned aggregate {} while paging replay aggregate {}", + event.aggregate_id(), + aggregate_id + ); + } + if event.seq() != expected_sequence { + bail!( + "EventStore replay sequence gap for aggregate {}: expected {}, got {}", + aggregate_id, + expected_sequence, + event.seq() + ); + } + expected_sequence = expected_sequence + .checked_add(1) + .context("EventStore replay sequence overflow")?; + max_timestamp = Some(max_timestamp.map_or(event.ts(), |ts| ts.max(event.ts()))); + } + + cursor = expected_sequence; + total_events = total_events + .checked_add(page.len()) + .context("EventStore replay event count overflow")?; + let page_was_full = page.len() == REPLAY_QUERY_PAGE_SIZE; + page.sort_by_key(event_order_key); + runs.push(write_run(page)?); + + if !page_was_full { + break; + } + } + } + + let runs = compact_runs(runs)?; + Ok(Self { + runs, + total_events, + max_timestamp, + }) + } + + pub(crate) fn total_events(&self) -> usize { + self.total_events + } + + pub(crate) async fn replay(self, bus: &BusHandle) -> Result { + if let Some(max_timestamp) = self.max_timestamp { + bus.seed_clock(max_timestamp)?; + } + + let total_events = self.total_events; + let mut replayed = 0usize; + let mut merger = RunMerger::new(&self.runs)?; + while let Some(event) = merger.next_event()? { + if SyncPlanner::classify_replay(&event) == ReplayDecision::SkipInfrastructure { + continue; + } + bus.event_bus().send(EventBusFanout(event)).await??; + replayed += 1; + if replayed.is_multiple_of(REPLAY_PROGRESS_INTERVAL) { + info!( + replayed_events = replayed, + total_events, "EventStore replay progress" + ); + } + } + bus.event_bus().send(EventBusBarrier).await?; + Ok(replayed) + } +} + +async fn query_page( + eventstore: &Recipient>, + aggregate_id: AggregateId, + cursor: u64, +) -> Result> { + let (addr, rx) = actix_toolbox::oneshot::(); + eventstore + .send( + EventStoreQueryBy::::new( + CorrelationId::new(), + std::collections::HashMap::from([(aggregate_id, cursor)]), + addr, + ) + .with_limit(REPLAY_QUERY_PAGE_SIZE as u64), + ) + .await + .context("EventStore router stopped during paged replay")?; + Ok(rx + .await + .context("EventStore did not return a paged replay response")? + .into_events()) +} + +fn event_order_key(event: &InterfoldEvent) -> (u128, AggregateId, u64) { + (event.ts(), event.aggregate_id(), event.seq()) +} + +fn write_run(events: Vec) -> Result { + let mut file = NamedTempFile::new().context("failed to create EventStore replay spool file")?; + { + let mut writer = BufWriter::new(file.as_file_mut()); + for event in events { + write_event(&mut writer, &event)?; + } + writer.flush().context("failed to flush replay spool run")?; + } + Ok(file) +} + +fn compact_runs(mut runs: Vec) -> Result> { + while runs.len() > REPLAY_MERGE_FAN_IN { + let mut next = Vec::with_capacity(runs.len().div_ceil(REPLAY_MERGE_FAN_IN)); + let mut iter = runs.into_iter(); + loop { + let group: Vec<_> = iter.by_ref().take(REPLAY_MERGE_FAN_IN).collect(); + if group.is_empty() { + break; + } + if group.len() == 1 { + next.extend(group); + } else { + next.push(merge_to_run(&group)?); + } + } + runs = next; + } + Ok(runs) +} + +fn merge_to_run(runs: &[NamedTempFile]) -> Result { + let mut output = NamedTempFile::new().context("failed to create merged replay spool file")?; + { + let mut writer = BufWriter::new(output.as_file_mut()); + let mut merger = RunMerger::new(runs)?; + while let Some(event) = merger.next_event()? { + write_event(&mut writer, &event)?; + } + writer + .flush() + .context("failed to flush merged replay run")?; + } + Ok(output) +} + +fn write_event(writer: &mut impl Write, event: &InterfoldEvent) -> Result<()> { + let bytes = bincode::serialize(event).context("failed to encode replay spool event")?; + if bytes.len() > MAX_SPOOLED_EVENT_BYTES { + bail!( + "replay event encoded to {} bytes, exceeding spool record limit {}", + bytes.len(), + MAX_SPOOLED_EVENT_BYTES + ); + } + let len = u64::try_from(bytes.len()).context("replay event length does not fit u64")?; + writer + .write_all(&len.to_le_bytes()) + .and_then(|_| writer.write_all(&bytes)) + .context("failed to write replay spool event") +} + +fn read_event(reader: &mut impl Read) -> Result> { + let mut len_bytes = [0u8; 8]; + match reader + .read(&mut len_bytes[..1]) + .context("failed to read replay spool length")? + { + 0 => return Ok(None), + 1 => reader + .read_exact(&mut len_bytes[1..]) + .context("truncated replay spool length prefix")?, + _ => unreachable!("single-byte read returned more than one byte"), + } + let len = usize::try_from(u64::from_le_bytes(len_bytes)) + .context("replay spool record length does not fit usize")?; + if len > MAX_SPOOLED_EVENT_BYTES { + bail!( + "replay spool record length {} exceeds limit {}", + len, + MAX_SPOOLED_EVENT_BYTES + ); + } + let mut bytes = vec![0u8; len]; + reader + .read_exact(&mut bytes) + .context("truncated replay spool event")?; + bincode::deserialize(&bytes) + .context("failed to decode replay spool event") + .map(Some) +} + +struct HeapItem { + key: (u128, AggregateId, u64), + run: usize, + event: InterfoldEvent, +} + +impl PartialEq for HeapItem { + fn eq(&self, other: &Self) -> bool { + self.key == other.key && self.run == other.run + } +} + +impl Eq for HeapItem {} + +impl PartialOrd for HeapItem { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for HeapItem { + fn cmp(&self, other: &Self) -> Ordering { + other + .key + .cmp(&self.key) + .then_with(|| other.run.cmp(&self.run)) + } +} + +struct RunMerger { + readers: Vec>, + heap: BinaryHeap, +} + +impl RunMerger { + fn new(runs: &[NamedTempFile]) -> Result { + let mut readers = Vec::with_capacity(runs.len()); + let mut heap = BinaryHeap::new(); + for (run, file) in runs.iter().enumerate() { + let mut reader = BufReader::new( + file.reopen() + .context("failed to reopen EventStore replay spool run")?, + ); + if let Some(event) = read_event(&mut reader)? { + heap.push(HeapItem { + key: event_order_key(&event), + run, + event, + }); + } + readers.push(reader); + } + Ok(Self { readers, heap }) + } + + fn next_event(&mut self) -> Result> { + let Some(item) = self.heap.pop() else { + return Ok(None); + }; + if let Some(event) = read_event(&mut self.readers[item.run])? { + self.heap.push(HeapItem { + key: event_order_key(&event), + run: item.run, + event, + }); + } + Ok(Some(item.event)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use e3_ciphernode_builder::EventSystem; + use e3_events::{E3id, EventPublisher, Sequenced, TestEvent}; + + fn event(aggregate: u64, sequence: u64, timestamp: u128) -> InterfoldEvent { + InterfoldEvent::::test_event("spooled") + .e3_id(E3id::new(sequence.to_string(), aggregate)) + .seq(sequence) + .ts(timestamp) + .build() + } + + #[test] + fn merge_orders_multiple_bounded_runs_deterministically() -> Result<()> { + let first = write_run(vec![event(1, 2, 30), event(1, 3, 50)])?; + let second = write_run(vec![event(2, 1, 10), event(2, 2, 40)])?; + let third = write_run(vec![event(1, 1, 20), event(3, 1, 40)])?; + let mut merger = RunMerger::new(&[first, second, third])?; + let mut keys = Vec::new(); + while let Some(event) = merger.next_event()? { + keys.push(event_order_key(&event)); + } + + assert_eq!( + keys, + vec![ + (10, AggregateId::new(2), 1), + (20, AggregateId::new(1), 1), + (30, AggregateId::new(1), 2), + (40, AggregateId::new(2), 2), + (40, AggregateId::new(3), 1), + (50, AggregateId::new(1), 3), + ] + ); + Ok(()) + } + + #[test] + fn fresh_snapshot_cursor_starts_at_first_one_based_log_sequence() { + assert_eq!(first_replay_sequence(0), 1); + assert_eq!(first_replay_sequence(7), 7); + } + + #[actix::test] + async fn load_pages_a_fresh_log_larger_than_one_query() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("replay-spool-paging"); + let count = REPLAY_QUERY_PAGE_SIZE + 1; + for index in 0..count { + bus.publish_without_context(TestEvent::new("paging", index as u64))?; + } + bus.flush_event_pipeline().await?; + + let eventstore = system.eventstore_reader()?.seq(); + let spool = ReplaySpool::load( + &eventstore, + std::collections::HashMap::from([(AggregateId::new(0), 0)]), + ) + .await?; + + assert_eq!(spool.total_events(), count); + assert_eq!(spool.runs.len(), 2); + Ok(()) + } +} From 603256775197f8ef9ddf8edceb032ae01a09c4a8 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 13 Jul 2026 21:28:34 +0500 Subject: [PATCH 52/62] fix(net): upgrade libp2p transport stack --- Cargo.lock | 462 +++++++++++++++----------------- Cargo.toml | 3 +- crates/net/src/net_interface.rs | 30 ++- 3 files changed, 241 insertions(+), 254 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a1bd56c50a..64fb299671 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -759,7 +759,7 @@ dependencies = [ "either", "futures", "futures-utils-wasm", - "lru 0.13.0", + "lru", "parking_lot 0.12.5", "pin-project", "reqwest", @@ -1597,9 +1597,9 @@ dependencies = [ [[package]] name = "asn1-rs" -version = "0.6.2" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5493c3bedbacf7fd7382c6346bbd66687d12bbaad3a89a2d2c303ee6cf20b048" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" dependencies = [ "asn1-rs-derive", "asn1-rs-impl", @@ -1607,15 +1607,15 @@ dependencies = [ "nom", "num-traits", "rusticata-macros", - "thiserror 1.0.69", + "thiserror 2.0.18", "time", ] [[package]] name = "asn1-rs-derive" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "965c2d33e53cb6b267e148a4cb0760bc01f4904c1cd4bb4002a085bb016d1490" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" dependencies = [ "proc-macro2", "quote", @@ -1843,11 +1843,12 @@ checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] name = "attohttpc" -version = "0.24.1" +version = "0.30.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9a9bf8b79a749ee0b911b91b671cc2b6c670bdbc7e3dfd537576ddc94bb2a2" +checksum = "16e2cdb6d5ed835199484bb92bb8b3edd526effe995c61732580439c1a67e2e9" dependencies = [ - "http 0.2.12", + "base64", + "http 1.4.0", "log", "url", ] @@ -2659,6 +2660,12 @@ dependencies = [ "itertools 0.10.5", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -2847,9 +2854,9 @@ dependencies = [ [[package]] name = "der-parser" -version = "9.0.0" +version = "10.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cd0a5c643689626bec213c4d8bd4d96acc8ffdb4ad4bb6bc16abf27d5f4b553" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" dependencies = [ "asn1-rs", "displaydoc", @@ -4257,6 +4264,18 @@ dependencies = [ "once_cell", ] +[[package]] +name = "fastbloom" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" +dependencies = [ + "getrandom 0.3.4", + "libm", + "rand 0.9.2", + "siphasher", +] + [[package]] name = "fastrand" version = "2.3.0" @@ -4615,17 +4634,6 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" -[[package]] -name = "futures-ticker" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9763058047f713632a52e916cc7f6a4b3fc6e9fc1ff8c5b1dc49e5a89041682e" -dependencies = [ - "futures", - "futures-timer", - "instant", -] - [[package]] name = "futures-timer" version = "3.0.3" @@ -4698,11 +4706,22 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", "wasm-bindgen", ] +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", +] + [[package]] name = "ghash" version = "0.5.1" @@ -4832,6 +4851,9 @@ name = "hashbrown" version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", +] [[package]] name = "hashbrown" @@ -4855,6 +4877,24 @@ dependencies = [ "serde_core", ] +[[package]] +name = "hashlink" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ba4ff7128dee98c7dc9794b6a411377e1404dba1c97deb8d1a55297bd25d8af" +dependencies = [ + "hashbrown 0.14.5", +] + +[[package]] +name = "hashlink" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1" +dependencies = [ + "hashbrown 0.15.5", +] + [[package]] name = "heck" version = "0.4.1" @@ -4896,9 +4936,9 @@ checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" [[package]] name = "hickory-proto" -version = "0.24.4" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92652067c9ce6f66ce53cc38d1169daa36e6e7eb7dd3b63b5103bd9d97117248" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" dependencies = [ "async-trait", "cfg-if", @@ -4910,9 +4950,10 @@ dependencies = [ "idna", "ipnet", "once_cell", - "rand 0.8.5", + "rand 0.9.2", + "ring", "socket2 0.5.10", - "thiserror 1.0.69", + "thiserror 2.0.18", "tinyvec", "tokio", "tracing", @@ -4921,21 +4962,21 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.4" +version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbb117a1ca520e111743ab2f6688eddee69db4e0ea242545a604dce8a66fd22e" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" dependencies = [ "cfg-if", "futures-util", "hickory-proto", "ipconfig", - "lru-cache", + "moka", "once_cell", "parking_lot 0.12.5", - "rand 0.8.5", + "rand 0.9.2", "resolv-conf", "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.18", "tokio", "tracing", ] @@ -4979,17 +5020,6 @@ dependencies = [ "itoa", ] -[[package]] -name = "http-body" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" -dependencies = [ - "bytes", - "http 0.2.12", - "pin-project-lite", -] - [[package]] name = "http-body" version = "1.0.1" @@ -5009,7 +5039,7 @@ dependencies = [ "bytes", "futures-core", "http 1.4.0", - "http-body 1.0.1", + "http-body", "pin-project-lite", ] @@ -5025,30 +5055,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" -[[package]] -name = "hyper" -version = "0.14.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41dfc780fdec9373c01bae43289ea34c972e40ee3c9f6b3c8801a35f35586ce7" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2 0.3.27", - "http 0.2.12", - "http-body 0.4.6", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.5.10", - "tokio", - "tower-service", - "tracing", - "want", -] - [[package]] name = "hyper" version = "1.8.1" @@ -5061,7 +5067,7 @@ dependencies = [ "futures-core", "h2 0.4.13", "http 1.4.0", - "http-body 1.0.1", + "http-body", "httparse", "itoa", "pin-project-lite", @@ -5078,7 +5084,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ "http 1.4.0", - "hyper 1.8.1", + "hyper", "hyper-util", "rustls", "rustls-pki-types", @@ -5094,7 +5100,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" dependencies = [ - "hyper 1.8.1", + "hyper", "hyper-util", "pin-project-lite", "tokio", @@ -5109,7 +5115,7 @@ checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" dependencies = [ "bytes", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-util", "native-tls", "tokio", @@ -5129,8 +5135,8 @@ dependencies = [ "futures-core", "futures-util", "http 1.4.0", - "http-body 1.0.1", - "hyper 1.8.1", + "http-body", + "hyper", "ipnet", "libc", "percent-encoding", @@ -5310,18 +5316,20 @@ dependencies = [ [[package]] name = "igd-next" -version = "0.14.3" +version = "0.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "064d90fec10d541084e7b39ead8875a5a80d9114a2b18791565253bae25f49e4" +checksum = "516893339c97f6011282d5825ac94fc1c7aad5cad26bdc2d0cee068c0bf97f97" dependencies = [ "async-trait", "attohttpc", "bytes", "futures", - "http 0.2.12", - "hyper 0.14.32", + "http 1.4.0", + "http-body-util", + "hyper", + "hyper-util", "log", - "rand 0.8.5", + "rand 0.9.2", "tokio", "url", "xmltree", @@ -5740,9 +5748,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libp2p" -version = "0.54.1" +version = "0.56.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbbe80f9c7e00526cd6b838075b9c171919404a4732cb2fa8ece0a093223bfc4" +checksum = "ce71348bf5838e46449ae240631117b487073d5f347c06d434caddcb91dceb5a" dependencies = [ "bytes", "either", @@ -5768,38 +5776,36 @@ dependencies = [ "multiaddr", "pin-project", "rw-stream-sink", - "thiserror 1.0.69", + "thiserror 2.0.18", ] [[package]] name = "libp2p-allow-block-list" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1027ccf8d70320ed77e984f273bc8ce952f623762cb9bf2d126df73caef8041" +checksum = "d16ccf824ee859ca83df301e1c0205270206223fd4b1f2e512a693e1912a8f4a" dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "void", ] [[package]] name = "libp2p-connection-limits" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d003540ee8baef0d254f7b6bfd79bac3ddf774662ca0abf69186d517ef82ad8" +checksum = "a18b8b607cf3bfa2f8c57db9c7d8569a315d5cc0a282e6bfd5ebfc0a9840b2a0" dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "void", ] [[package]] name = "libp2p-core" -version = "0.42.0" +version = "0.43.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a61f26c83ed111104cd820fe9bc3aaabbac5f1652a1d213ed6e900b7918a1298" +checksum = "249128cd37a2199aff30a7675dffa51caf073b51aa612d2f544b19932b9aebca" dependencies = [ "either", "fnv", @@ -5809,25 +5815,22 @@ dependencies = [ "multiaddr", "multihash", "multistream-select", - "once_cell", "parking_lot 0.12.5", "pin-project", "quick-protobuf", "rand 0.8.5", "rw-stream-sink", - "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.18", "tracing", "unsigned-varint 0.8.0", - "void", "web-time", ] [[package]] name = "libp2p-dns" -version = "0.42.0" +version = "0.44.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97f37f30d5c7275db282ecd86e54f29dd2176bd3ac656f06abf43bedb21eb8bd" +checksum = "0b770c1c8476736ca98c578cba4b505104ff8e842c2876b528925f9766379f9a" dependencies = [ "async-trait", "futures", @@ -5841,10 +5844,11 @@ dependencies = [ [[package]] name = "libp2p-gossipsub" -version = "0.47.0" +version = "0.49.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4e830fdf24ac8c444c12415903174d506e1e077fbe3875c404a78c5935a8543" +checksum = "a538e571cd38f504f761c61b8f79127489ea7a7d6f05c41ca15d31ffb5726326" dependencies = [ + "async-channel 2.5.0", "asynchronous-codec", "base64", "byteorder", @@ -5852,29 +5856,27 @@ dependencies = [ "either", "fnv", "futures", - "futures-ticker", + "futures-timer", "getrandom 0.2.17", + "hashlink 0.9.1", "hex_fmt", "libp2p-core", "libp2p-identity", "libp2p-swarm", - "prometheus-client", "quick-protobuf", "quick-protobuf-codec", "rand 0.8.5", "regex", "sha2", - "smallvec", "tracing", - "void", "web-time", ] [[package]] name = "libp2p-identify" -version = "0.45.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1711b004a273be4f30202778856368683bd9a83c4c7dcc8f848847606831a4e3" +checksum = "8ab792a8b68fdef443a62155b01970c81c3aadab5e659621b063ef252a8e65e8" dependencies = [ "asynchronous-codec", "either", @@ -5884,13 +5886,11 @@ dependencies = [ "libp2p-core", "libp2p-identity", "libp2p-swarm", - "lru 0.12.5", "quick-protobuf", "quick-protobuf-codec", "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.18", "tracing", - "void", ] [[package]] @@ -5913,11 +5913,10 @@ dependencies = [ [[package]] name = "libp2p-kad" -version = "0.46.2" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced237d0bd84bbebb7c2cad4c073160dacb4fe40534963c32ed6d4c6bb7702a3" +checksum = "13d3fd632a5872ec804d37e7413ceea20588f69d027a0fa3c46f82574f4dee60" dependencies = [ - "arrayvec", "asynchronous-codec", "bytes", "either", @@ -5933,20 +5932,18 @@ dependencies = [ "rand 0.8.5", "sha2", "smallvec", - "thiserror 1.0.69", + "thiserror 2.0.18", "tracing", - "uint", - "void", + "uint 0.10.0", "web-time", ] [[package]] name = "libp2p-mdns" -version = "0.46.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14b8546b6644032565eb29046b42744aee1e9f261ed99671b2c93fb140dba417" +checksum = "c66872d0f1ffcded2788683f76931be1c52e27f343edb93bc6d0bcd8887be443" dependencies = [ - "data-encoding", "futures", "hickory-proto", "if-watch", @@ -5958,14 +5955,13 @@ dependencies = [ "socket2 0.5.10", "tokio", "tracing", - "void", ] [[package]] name = "libp2p-metrics" -version = "0.15.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ebafa94a717c8442d8db8d3ae5d1c6a15e30f2d347e0cd31d057ca72e42566" +checksum = "805a555148522cb3414493a5153451910cb1a146c53ffbf4385708349baf62b7" dependencies = [ "futures", "libp2p-core", @@ -5982,11 +5978,10 @@ dependencies = [ [[package]] name = "libp2p-ping" -version = "0.45.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "005a34420359223b974ee344457095f027e51346e992d1e0dcd35173f4cdd422" +checksum = "74bb7fcdfd9fead4144a3859da0b49576f171a8c8c7c0bfc7c541921d25e60d3" dependencies = [ - "either", "futures", "futures-timer", "libp2p-core", @@ -5994,45 +5989,42 @@ dependencies = [ "libp2p-swarm", "rand 0.8.5", "tracing", - "void", "web-time", ] [[package]] name = "libp2p-quic" -version = "0.11.1" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46352ac5cd040c70e88e7ff8257a2ae2f891a4076abad2c439584a31c15fd24e" +checksum = "9dcc597d70bf7f6f30cbe07081802c836184e48416e89e9ce73a0ba2c56a319e" dependencies = [ - "bytes", "futures", "futures-timer", "if-watch", "libp2p-core", "libp2p-identity", "libp2p-tls", - "parking_lot 0.12.5", "quinn", + "quinn-proto", "rand 0.8.5", - "ring 0.17.14", + "ring", "rustls", "socket2 0.5.10", - "thiserror 1.0.69", + "thiserror 2.0.18", "tokio", "tracing", ] [[package]] name = "libp2p-request-response" -version = "0.27.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1356c9e376a94a75ae830c42cdaea3d4fe1290ba409a22c809033d1b7dcab0a6" +checksum = "a9f1cca83488b90102abac7b67d5c36fc65bc02ed47620228af7ed002e6a1478" dependencies = [ "async-trait", "cbor4ii", "futures", "futures-bounded", - "futures-timer", "libp2p-core", "libp2p-identity", "libp2p-swarm", @@ -6040,87 +6032,81 @@ dependencies = [ "serde", "smallvec", "tracing", - "void", - "web-time", ] [[package]] name = "libp2p-swarm" -version = "0.45.1" +version = "0.47.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7dd6741793d2c1fb2088f67f82cf07261f25272ebe3c0b0c311e0c6b50e851a" +checksum = "ce88c6c4bf746c8482480345ea3edfd08301f49e026889d1cbccfa1808a9ed9e" dependencies = [ "either", "fnv", "futures", "futures-timer", + "hashlink 0.10.0", "libp2p-core", "libp2p-identity", "libp2p-swarm-derive", - "lru 0.12.5", "multistream-select", - "once_cell", "rand 0.8.5", "smallvec", "tokio", "tracing", - "void", "web-time", ] [[package]] name = "libp2p-swarm-derive" -version = "0.35.0" +version = "0.35.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "206e0aa0ebe004d778d79fb0966aa0de996c19894e2c0605ba2f8524dd4443d8" +checksum = "dd297cf53f0cb3dee4d2620bb319ae47ef27c702684309f682bdb7e55a18ae9c" dependencies = [ "heck 0.5.0", - "proc-macro2", "quote", "syn 2.0.116", ] [[package]] name = "libp2p-tcp" -version = "0.42.0" +version = "0.44.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad964f312c59dcfcac840acd8c555de8403e295d39edf96f5240048b5fcaa314" +checksum = "fb6585b9309699f58704ec9ab0bb102eca7a3777170fa91a8678d73ca9cafa93" dependencies = [ "futures", "futures-timer", "if-watch", "libc", "libp2p-core", - "libp2p-identity", - "socket2 0.5.10", + "socket2 0.6.2", "tokio", "tracing", ] [[package]] name = "libp2p-tls" -version = "0.5.0" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47b23dddc2b9c355f73c1e36eb0c3ae86f7dc964a3715f0731cfad352db4d847" +checksum = "96ff65a82e35375cbc31ebb99cacbbf28cb6c4fefe26bf13756ddcf708d40080" dependencies = [ "futures", "futures-rustls", "libp2p-core", "libp2p-identity", "rcgen", - "ring 0.17.14", + "ring", "rustls", - "rustls-webpki 0.101.7", - "thiserror 1.0.69", + "rustls-webpki", + "thiserror 2.0.18", "x509-parser", "yasna", ] [[package]] name = "libp2p-upnp" -version = "0.3.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01bf2d1b772bd3abca049214a3304615e6a36fa6ffc742bdd1ba774486200b8f" +checksum = "4757e65fe69399c1a243bbb90ec1ae5a2114b907467bf09f3575e899815bb8d3" dependencies = [ "futures", "futures-timer", @@ -6129,7 +6115,6 @@ dependencies = [ "libp2p-swarm", "tokio", "tracing", - "void", ] [[package]] @@ -6181,12 +6166,6 @@ dependencies = [ "thiserror 1.0.69", ] -[[package]] -name = "linked-hash-map" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" - [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -6234,15 +6213,6 @@ dependencies = [ "value-bag", ] -[[package]] -name = "lru" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" -dependencies = [ - "hashbrown 0.15.5", -] - [[package]] name = "lru" version = "0.13.0" @@ -6252,15 +6222,6 @@ dependencies = [ "hashbrown 0.15.5", ] -[[package]] -name = "lru-cache" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" -dependencies = [ - "linked-hash-map", -] - [[package]] name = "lru-slab" version = "0.1.2" @@ -6357,6 +6318,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "moka" +version = "0.12.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957228ad12042ee839f93c8f257b62b4c0ab5eaae1d4fa60de53b27c9d7c5046" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot 0.12.5", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + [[package]] name = "multiaddr" version = "0.18.2" @@ -6916,9 +6894,9 @@ dependencies = [ [[package]] name = "oid-registry" -version = "0.7.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d8034d9489cdaf79228eb9f6a3b8d7bb32ba00d6645ebd48eef4077ceb5bd9" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" dependencies = [ "asn1-rs", ] @@ -6928,6 +6906,10 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -7568,7 +7550,7 @@ checksum = "0b34d9fd68ae0b74a41b21c03c2f62847aa0ffea044eee893b4c140b37e244e2" dependencies = [ "fixed-hash", "impl-codec", - "uint", + "uint 0.9.5", ] [[package]] @@ -7626,9 +7608,9 @@ dependencies = [ [[package]] name = "prometheus-client" -version = "0.22.3" +version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "504ee9ff529add891127c4827eb481bd69dc0ebc72e9a682e187db4caa60c3ca" +checksum = "cf41c1a7c32ed72abe5082fb19505b969095c12da9f5732a4bc9878757fd087c" dependencies = [ "dtoa", "itoa", @@ -7912,10 +7894,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", + "fastbloom", "getrandom 0.3.4", "lru-slab", "rand 0.9.2", - "ring 0.17.14", + "ring", "rustc-hash", "rustls", "rustls-pki-types", @@ -7955,6 +7938,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "radium" version = "0.7.0" @@ -8103,12 +8092,13 @@ dependencies = [ [[package]] name = "rcgen" -version = "0.11.3" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52c4f3084aa3bc7dfbba4eff4fab2a54db4324965d8872ab933565e6fbd83bc6" +checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2" dependencies = [ "pem", - "ring 0.16.20", + "ring", + "rustls-pki-types", "time", "yasna", ] @@ -8243,9 +8233,9 @@ dependencies = [ "futures-util", "h2 0.4.13", "http 1.4.0", - "http-body 1.0.1", + "http-body", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-rustls", "hyper-tls", "hyper-util", @@ -8293,21 +8283,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "ring" -version = "0.16.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" -dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted 0.7.1", - "web-sys", - "winapi", -] - [[package]] name = "ring" version = "0.17.14" @@ -8318,7 +8293,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.17", "libc", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] @@ -8508,9 +8483,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ "once_cell", - "ring 0.17.14", + "ring", "rustls-pki-types", - "rustls-webpki 0.103.13", + "rustls-webpki", "subtle", "zeroize", ] @@ -8525,25 +8500,15 @@ dependencies = [ "zeroize", ] -[[package]] -name = "rustls-webpki" -version = "0.101.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" -dependencies = [ - "ring 0.17.14", - "untrusted 0.9.0", -] - [[package]] name = "rustls-webpki" version = "0.103.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" dependencies = [ - "ring 0.17.14", + "ring", "rustls-pki-types", - "untrusted 0.9.0", + "untrusted", ] [[package]] @@ -9183,12 +9148,6 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bceb57dc07c92cdae60f5b27b3fa92ecaaa42fe36c55e22dbfb0b44893e0b1f7" -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "spki" version = "0.7.3" @@ -9351,6 +9310,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "tap" version = "1.0.1" @@ -9732,9 +9697,9 @@ dependencies = [ "base64", "bytes", "http 1.4.0", - "http-body 1.0.1", + "http-body", "http-body-util", - "hyper 1.8.1", + "hyper", "hyper-timeout", "hyper-util", "percent-encoding", @@ -9793,7 +9758,7 @@ dependencies = [ "bytes", "futures-util", "http 1.4.0", - "http-body 1.0.1", + "http-body", "iri-string", "pin-project-lite", "tower 0.5.3", @@ -9974,6 +9939,18 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "uint" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909988d098b2f738727b161a106cfc7cab00c539c2687a8836f8e565976fb53e" +dependencies = [ + "byteorder", + "crunchy", + "hex", + "static_assertions", +] + [[package]] name = "unarray" version = "0.1.4" @@ -10053,12 +10030,6 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb066959b24b5196ae73cb057f45598450d2c5f71460e98c49b738086eff9c06" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -10100,6 +10071,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea5fab0d6c3c01ae70085a09cb03d4c7a1d6314e2b3e075392783396d724ca0a" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -10158,12 +10140,6 @@ version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" -[[package]] -name = "void" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" - [[package]] name = "wait-timeout" version = "0.2.1" @@ -10774,9 +10750,9 @@ dependencies = [ [[package]] name = "x509-parser" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcbc162f30700d6f3f82a24bf7cc62ffe7caea42c0b2cba8bf7f3ae50cf51f69" +checksum = "4569f339c0c402346d4a75a9e39cf8dad310e287eef1ff56d4c68e5067f53460" dependencies = [ "asn1-rs", "data-encoding", @@ -10785,7 +10761,7 @@ dependencies = [ "nom", "oid-registry", "rusticata-macros", - "thiserror 1.0.69", + "thiserror 2.0.18", "time", ] diff --git a/Cargo.toml b/Cargo.toml index 9918edae3c..e5c97b1501 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -206,10 +206,11 @@ tracing-test = { version = "=0.2.5", features = ["no-env-filter"] } url = "=2.5.4" vfs = { git = "https://github.com/ryardley/rust-vfs.git", features = ["async-vfs"] } libc = "=0.2.180" -libp2p = { version = "=0.54.1", features = [ +libp2p = { version = "=0.56.0", features = [ "gossipsub", "identify", "dns", + "ed25519", "kad", "macros", "mdns", diff --git a/crates/net/src/net_interface.rs b/crates/net/src/net_interface.rs index 4be72a78b0..7ca7145661 100644 --- a/crates/net/src/net_interface.rs +++ b/crates/net/src/net_interface.rs @@ -404,7 +404,7 @@ async fn process_swarm_event( if let Some(ref failed_peer) = peer_id { if let DialError::WrongPeerId { obtained, - ref endpoint, + ref address, } = error { // The node at this address has a new PeerId (e.g. restarted with new keys). @@ -412,7 +412,7 @@ async fn process_swarm_event( // The stale entry can be re-learned from other peers' routing tables, // so repeats are expected: handle them quietly (debug, no bootstrap) // to avoid flooding the logs and re-fueling the dial loop. - let remote_addr = endpoint.get_remote_address().clone(); + let remote_addr = address.clone(); let mismatch_count = peer_failures.identity_mismatch.record_failure(failed_peer); // The stale ID is being removed from the routing table, so its @@ -609,6 +609,7 @@ async fn process_swarm_event( SwarmEvent::Behaviour(NodeBehaviourEvent::RequestResponse( RequestResponseEvent::Message { peer, + connection_id, message: RequestResponseMessage::Request { request, @@ -618,8 +619,8 @@ async fn process_swarm_event( }, )) => { debug!( - "Incoming request received (peer={}, id={})", - peer, request_id + "Incoming request received (peer={}, connection={}, id={})", + peer, connection_id, request_id ); let responder = DirectResponder::new(request_id, ChannelType::Channel(channel), cmd_tx) .with_request(request); @@ -656,13 +657,14 @@ async fn process_swarm_event( SwarmEvent::Behaviour(NodeBehaviourEvent::RequestResponse( RequestResponseEvent::OutboundFailure { peer, + connection_id, request_id, error, }, )) => { warn!( - "Outbound request failed: peer={}, id={}, error={:?}", - peer, request_id, error + "Outbound request failed: peer={}, connection={}, id={}, error={:?}", + peer, connection_id, request_id, error ); let correlation_id = correlator.expire(request_id)?; event_tx.send(NetEvent::OutgoingRequestFailed(OutgoingRequestFailed { @@ -674,20 +676,28 @@ async fn process_swarm_event( SwarmEvent::Behaviour(NodeBehaviourEvent::RequestResponse( RequestResponseEvent::InboundFailure { peer, + connection_id, request_id, error, }, )) => { warn!( - "Inbound request failed: peer={}, id={}, error={:?}", - peer, request_id, error + "Inbound request failed: peer={}, connection={}, id={}, error={:?}", + peer, connection_id, request_id, error ); } SwarmEvent::Behaviour(NodeBehaviourEvent::RequestResponse( - RequestResponseEvent::ResponseSent { peer, request_id }, + RequestResponseEvent::ResponseSent { + peer, + connection_id, + request_id, + }, )) => { - debug!("Response sent to peer={}, id={}", peer, request_id); + debug!( + "Response sent to peer={}, connection={}, id={}", + peer, connection_id, request_id + ); } SwarmEvent::Behaviour(NodeBehaviourEvent::Identify( From fa0deb3b7455a217bc325201f773e9772f7656f6 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Mon, 13 Jul 2026 22:46:54 +0500 Subject: [PATCH 53/62] refactor(builder): construct node handles with named fields --- crates/ciphernode-builder/src/ciphernode.rs | 26 ------------------- .../src/ciphernode_builder.rs | 10 +++---- 2 files changed, 5 insertions(+), 31 deletions(-) diff --git a/crates/ciphernode-builder/src/ciphernode.rs b/crates/ciphernode-builder/src/ciphernode.rs index 4cfe87e86d..136170a592 100644 --- a/crates/ciphernode-builder/src/ciphernode.rs +++ b/crates/ciphernode-builder/src/ciphernode.rs @@ -79,32 +79,6 @@ impl PartialEq for CiphernodeHandle { impl Eq for CiphernodeHandle {} impl CiphernodeHandle { - pub fn new( - address: String, - store: DataStore, - bus: BusHandle, - history: Option>>, - errors: Option>>, - peer_id: PeerId, - net_interface: NetInterfaceKind, - network_status: NetworkStatus, - eventstore: EventStoreReader, - aggregate_ids: Vec, - ) -> Self { - Self { - address, - store, - bus, - history, - errors, - peer_id, - net_interface, - network_status, - eventstore, - aggregate_ids, - } - } - pub fn bus(&self) -> &BusHandle { &self.bus } diff --git a/crates/ciphernode-builder/src/ciphernode_builder.rs b/crates/ciphernode-builder/src/ciphernode_builder.rs index 5923008519..5e965017a6 100644 --- a/crates/ciphernode-builder/src/ciphernode_builder.rs +++ b/crates/ciphernode-builder/src/ciphernode_builder.rs @@ -608,18 +608,18 @@ impl CiphernodeBuilder { net_buffer.wait_until_running(), )?; - Ok(CiphernodeHandle::new( - addr.to_owned(), + Ok(CiphernodeHandle { + address: addr.to_owned(), store, bus, history, errors, peer_id, - net_kind, + net_interface: net_kind, network_status, eventstore, - aggregate_config.indexed_ids(), - )) + aggregate_ids: aggregate_config.indexed_ids(), + }) } // ── build() sub-functions ────────────────────────────────────────── From 4ffe4fb011c7c925d61995c6a109eb58e226a147 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Sat, 11 Jul 2026 19:53:24 +0500 Subject: [PATCH 54/62] docs(crates): record final architecture and audit ledger --- agent/CRATES_ARCHITECTURE.md | 819 ++++++++++++++++++ agent/RULES.md | 3 + agent/flow-trace/00_INDEX.md | 24 +- agent/flow-trace/04_DKG_AND_COMPUTATION.md | 3 + .../06_DEACTIVATION_AND_COMPLETION.md | 87 +- 5 files changed, 898 insertions(+), 38 deletions(-) create mode 100644 agent/CRATES_ARCHITECTURE.md diff --git a/agent/CRATES_ARCHITECTURE.md b/agent/CRATES_ARCHITECTURE.md new file mode 100644 index 0000000000..20f892dd92 --- /dev/null +++ b/agent/CRATES_ARCHITECTURE.md @@ -0,0 +1,819 @@ +# Interfold Rust Workspace Architecture + +This document describes the implementation in `crates/`. It is intentionally +code-facing: names in the diagrams are crate, module, actor, message, or durable +repository names that can be searched in the workspace. Read it alongside the +prescriptive [`ARCHITECTURE.md`](ARCHITECTURE.md) contribution guide and +[`RULES.md`](RULES.md). + +## Dependency layers + +All 45 workspace packages are shown below. Arrows point from a dependent crate to a +direct dependency; the diagram keeps representative production edges rather than +reproducing every Cargo edge, while the groups record each crate's current primary +responsibility. Test-only edges are kept in the validation group. Several protocol +crates still import concrete infrastructure types; that boundary debt is recorded in +the audit rather than hidden by an aspirational diagram. + +```mermaid +flowchart TD + subgraph Entry[Entry, operations, and composition] + CLI[e3-cli] + EP[e3-entrypoint] + Builder[e3-ciphernode-builder] + Dash[e3-dashboard] + Daemon[e3-daemon-server] + Console[e3-console] + Init[e3-init] + Fs[e3-fs] + Support[e3-support-scripts] + Up[interfoldup] + end + + subgraph Clients[Client and compute surfaces] + ProgramServer[e3-program-server] + ComputeProvider[e3-compute-provider] + SDK[e3-sdk] + Indexer[e3-indexer] + EvmHelpers[e3-evm-helpers] + Wasm[e3-wasm] + end + + subgraph Workflows[Protocol workflows] + Request[e3-request] + Sortition[e3-sortition] + Keyshare[e3-keyshare] + Aggregator[e3-aggregator] + Slashing[e3-slashing] + Prover[e3-zk-prover] + Fhe[e3-fhe] + Multi[e3-multithread] + Trbfv[e3-trbfv] + BfvClient[e3-bfv-client] + end + + subgraph Adapters[Infrastructure adapters] + Evm[e3-evm] + Net[e3-net] + Sync[e3-sync] + Data[e3-data] + Logger[e3-logger] + end + + subgraph Vocabulary[Protocol vocabulary and deterministic rules] + Events[e3-events] + Config[e3-config] + FheParams[e3-fhe-params] + ZkHelpers[e3-zk-helpers] + CommitteeHash[e3-committee-hash] + end + + subgraph Foundations[Foundation crates] + Crypto[e3-crypto] + Poly[e3-polynomial] + Matrix[e3-parity-matrix] + Safe[e3-safe] + Hamt[e3-hamt] + Utils[e3-utils] + UtilsDerive[e3-utils-derive] + end + + subgraph Validation[Workspace validation] + TestHelpers[e3-test-helpers] + Tests[e3-tests] + end + + CLI --> EP + CLI --> Builder + Dash --> Builder + CLI --> Dash + CLI --> Daemon + CLI --> Init + CLI --> Support + EP --> Builder + Daemon --> Config + Daemon --> Console + Init --> Fs + Support --> Config + + ProgramServer --> ComputeProvider + ComputeProvider --> BfvClient + SDK --> BfvClient + SDK --> Indexer + SDK --> EvmHelpers + SDK --> FheParams + Indexer --> EvmHelpers + EvmHelpers --> Utils + Wasm --> BfvClient + Wasm --> FheParams + + Builder --> Aggregator + Builder --> Keyshare + Builder --> Request + Builder --> Sortition + Builder --> Slashing + Builder --> Prover + Builder --> Evm + Builder --> Net + Builder --> Sync + Builder --> Data + Builder --> Logger + + Aggregator --> Keyshare + Aggregator --> Prover + Aggregator --> Evm + Aggregator --> CommitteeHash + Keyshare --> Fhe + Keyshare --> Multi + Keyshare --> Trbfv + Slashing --> Request + Slashing --> ZkHelpers + Prover --> Slashing + Prover --> ZkHelpers + Multi --> Prover + Multi --> Trbfv + Fhe --> BfvClient + Trbfv --> BfvClient + BfvClient --> FheParams + BfvClient --> Poly + BfvClient --> ZkHelpers + + Evm --> Config + Evm --> Data + Evm --> Events + Net --> Config + Net --> Data + Net --> Events + Sync --> Config + Sync --> Data + Sync --> Events + Data --> Config + Data --> Events + Data --> Hamt + Logger --> Events + + Config --> Events + Config --> FheParams + Events --> Crypto + Events --> FheParams + Events --> Trbfv + Events --> ZkHelpers + ZkHelpers --> Poly + ZkHelpers --> Matrix + ZkHelpers --> Safe + Crypto --> Utils + Utils --> UtilsDerive + + Tests --> TestHelpers + TestHelpers --> Builder + TestHelpers --> SDK +``` + +The most important current boundary debt is that `e3-events` contains both neutral +event transport and rich protocol payloads that depend on cryptographic/FHE types. +Protocol workflow crates also depend directly on Actix and concrete repositories. +These are real constraints in the current code and are not papered over with empty +port traits. + +The resulting domain/application/adapter placement is module-level rather than a +clean crate boundary: + +```mermaid +flowchart LR + Composition[CiphernodeBuilder and entrypoint composition] --> App[Actix-backed protocol workflows] + Composition --> Adapters[concrete EVM, libp2p, storage, and proof adapters] + Composition --> Infra[Actix, RPC, libp2p, bb, stores, and task pools] + App --> Domain[deterministic domain modules and invariant types] + App --> Adapters + Adapters --> Infra + Domain --> Shared[e3-events payloads and transport types] + App --> Shared + Adapters --> Shared + + classDef debt fill:#fff1f0,stroke:#cf222e,color:#82071e + class App,Shared debt +``` + +Pure decision modules have no actor runtime (for example lifecycle transitions, sync +planning, network buffer decisions, document validation, accusation voting, and +aggregation state machines). Application coordination is still predominantly actor +handlers rather than independent service objects. Adapters are concrete and are wired +centrally by `CiphernodeBuilder`. Arrows in this diagram point from a consumer to what +it uses: domain code does not depend on the composition root, while application actors +still depend directly on concrete adapters in several crates. + +## Ciphernode construction and startup + +```mermaid +sequenceDiagram + participant OS as OS / CLI signal loop + participant CLI as e3-cli::start + participant Fence as ProcessFence + participant EP as e3-entrypoint::start + participant B as CiphernodeBuilder + participant ES as EventSystem + participant EV as EvmSystem + participant NET as e3-net + participant P as Protocol actors + participant SYNC as e3-sync::sync + + CLI->>Fence: acquire(db path, node name) + CLI->>EP: atomically create or reuse encrypted identities when autowallet is enabled + CLI->>EP: start(config, password) + EP->>EP: validate configuration and decrypt keys + EP->>B: configure stores, chains, signer, network, limits + B->>ES: create EventBus, Sequencer, EventStore, SnapshotBuffer + ES->>ES: rebuild timestamp index in bounded pages + B->>SYNC: schema preflight before state-writing actors + B->>EV: create per-chain readers, writers, and gateways + B->>P: install router, sortition, keyshare, proof, aggregation, slashing extensions + B->>NET: create libp2p interface and bounded startup buffer + B->>SYNC: replay, EVM backfill, network backfill + SYNC-->>P: EffectsEnabled then durable fanout fence + SYNC-->>P: reconciled history then durable fanout fence + SYNC-->>EV: SyncEnded then durable fanout fence + SYNC-->>B: startup complete + B-->>EP: CiphernodeHandle + EP-->>CLI: ready node +``` + +Startup has a configured outer deadline. The EVM and network startup buffers expose +readiness failures; a bound overflow fails startup instead of silently discarding +protocol observations. Effects remain disabled until durable replay and both +historical sources have been merged in HLC timestamp order. Schema preflight treats +only an empty key/value store or exactly the complete encrypted Ethereum/libp2p +identity pair as fresh. A partial identity, any additional unversioned key, any event +log without a marker, upgrades, and downgrades fail closed. This narrow exception is +required because autowallet atomically creates the two bootstrap identities before the +builder can stamp the schema; it does not let protocol state bypass compatibility checks. + +## Actor and message topology + +```mermaid +flowchart LR + Ext[EVM logs / libp2p bytes] --> Gateways[EVM gateways / NetEventBuffer] + Gateways --> Translate[typed translators] + Translate --> Handle[BusHandle admission and HLC] + Publishers[protocol publishers] --> Handle + Handle --> Seq[Sequencer] + Seq --> StoreRouter[EventStoreRouter] + StoreRouter --> Logs[(per-aggregate event logs)] + Logs --> StoreAck[StoreEventResponse] + StoreAck --> Seq + Seq --> Bus[EventBus] + + Bus --> Router[E3Router] + Bus --> Sortition[Sortition and selector] + Bus --> Proof[global proof request / verification actors] + Bus --> Committee[CommitteeFinalizer] + Bus --> Safety[per-E3 accusation / consistency actors] + Bus --> Writers[contract writers] + Bus --> Snapshot[SnapshotBuffer] + + Snapshot --> KV[(Sled repositories)] + Router --> Context[E3Context actor tree] + Context --> Keyshare[ThresholdKeyshare per E3] + Context --> Aggregators[PK and plaintext aggregators per E3] + Context --> Safety +``` + +Actors own scheduling, mailbox ordering, subscriptions, and lifecycle. Deterministic +decisions that have already been separated live in modules such as +`request::domain::lifecycle`, `sync::domain::sync_planner`, +`net::domain::net_buffer`, `slashing::domain::accusation_voting`, and the typed +aggregation state machines. The audit identifies handlers that still combine policy +with transport. The normal live path shown above is durable-before-dispatch, but it is +not fully backpressured: several `BusHandle`, `Sequencer`, `EventStoreRouter`, and +snapshot edges still use Actix `do_send`, which bypasses mailbox capacity. That debt is +called out explicitly below rather than representing the pipeline as end-to-end +bounded. ZK proof actors are composition-scoped EventBus subscribers. Threshold +keyshare and public-key/plaintext aggregation actors are request-scoped recipients +created by `E3Router` extensions and reached through `E3Context`; they are not direct +EventBus subscribers. Per-E3 accusation and consistency actors are context-owned but +also install direct subscriptions for the proof and slash events they consume. + +## Event ingestion, persistence, replay, and synchronization + +```mermaid +flowchart TD + subgraph LivePath[Live durable path] + Publish[BusHandle publish] --> Sequencer[Sequencer assigns per-aggregate sequence] + Sequencer -. do_send .-> EventStore[append event log and timestamp index] + EventStore -. response do_send .-> Sequencer + Sequencer -. do_send .-> Dispatch[EventBus dispatch] + Dispatch -->|await each live recipient| Subscribers[actor subscribers] + Dispatch --> SnapshotBuffer[aggregate snapshot buffer] + SnapshotBuffer -. do_send .-> SnapshotRouter[BatchRouter and per-sequence Batch actors] + SnapshotRouter --> Repositories[(Sled KV state)] + end + + subgraph Recovery[Restart and historical reconciliation] + Restart[restart] --> Index[reconcile timestamp index in 1024-record pages] + Index --> Schema[schema-version preflight before runtime actor writes] + Schema --> SnapshotMeta[load aggregate cursors and initial HLC floor] + SnapshotMeta --> Query[query every post-snapshot aggregate] + Query --> Runs[sort 1024-event pages into secure temporary runs] + Runs --> GlobalOrder[bounded-fan-in merge by HLC timestamp] + GlobalOrder --> ReplayFloor[advance HLC floor while loading the runs] + ReplayFloor -->|EventBus acknowledged fanout one event at a time| Dispatch + Dispatch -->|EventBusBarrier after completed fanout| EvmBackfill[configured-confirmation EVM backfill] + EvmBackfill --> NetBackfill[bounded historical network sync] + NetBackfill --> Merge[merge and sort EVM plus network history by HLC] + Merge --> Enable[EffectsEnabled] + Enable -->|durable pipeline and fanout fence| PersistHistory[persist and dispatch reconciled history] + PersistHistory -->|durable pipeline and fanout fence| End[SyncEnded] + End -->|durable pipeline and fanout fence| Live[live operation] + end + + classDef residual fill:#fff1f0,stroke:#cf222e,color:#82071e + class SnapshotRouter residual +``` + +The append-only event log is the durable source of truth. The timestamp index and +snapshots are derived state. EventStore construction reconciles missing index rows from +the log in strict 1,024-record pages and fail-stops on a gap or unusable index; it does +not decode the complete log solely to rebuild the index. +Post-snapshot events are queried per aggregate in 1,024-event pages and sorted into +secure temporary runs. Runs are compacted with bounded fan-in and merged globally by +persisted HLC timestamp, so memory and open-file use do not scale with the entire +backlog. Before fanout, the HLC floor advances to the maximum replay timestamp, which +covers a snapshot cursor stalled behind newer log records. Replay then waits for +concurrent acceptance by all current EventBus subscribers. An unavailable subscriber +or a subscriber blocked beyond the bounded acceptance timeout aborts recovery. An +`EventBusBarrier` therefore completes only after the last replay fanout has completed. +A persisted `Shutdown` event from the previous process is classified as infrastructure +and is not replayed into newly constructed actors. + +The EventBus mailbox remains bounded at `MAILBOX_LIMIT_LARGE` (2,560 messages). The +replay producer no longer attempts to enqueue the entire backlog into that mailbox in +one burst, and EventBus subscriber fanout no longer bypasses downstream mailbox limits. +EventStore query responses also await recipient capacity, preventing a full aggregation +mailbox from dropping one aggregate response and hanging startup. Recovery publishes +`EffectsEnabled`, canonical history, and `SyncEnded` as three separately fenced phases. +An event-log or timestamp-index write error panics the affected EventStore before live +dispatch. This preserves durable-before-dispatch safety, but under the default unwind +profile an Actix actor panic is contained at its spawned task boundary: it can kill the +store actor and stall the sequencer without terminating the process. Process-level +health supervision would need to detect the stalled pipeline, but the current runtime +does not provide that guarantee. A restart, when it occurs, treats the event log as +authoritative and reconciles missing derived index rows. + +Those replay guarantees bound local replay memory and file-descriptor use, but they do +not make the whole persistence path synchronously acknowledged. Live publication and +the sequencer/store response path still contain `do_send` edges. Snapshot replay also +forwards with `do_send`, and `BatchRouter` can retain one child `Batch` actor per open +aggregate/sequence until its timelock fires. A sufficiently large set of simultaneously +open snapshot batches can therefore create actors in proportion to that active set. + +## Networking-to-domain flow + +```mermaid +flowchart LR + Peer[remote PeerId] --> Quic[authenticated QUIC transport] + Quic --> Swarm[libp2p Swarm] + Swarm --> Signed[gossipsub signed message validation] + Signed --> Raw[bounded NetEvent broadcast] + Raw --> Startup[NetEventBuffer count + byte limits] + Raw --> SyncManager[NetSyncManager] + Startup -->|await actor acceptance after SyncEnded| Translator[NetEventTranslator] + Translator --> Allowlist{forwardable event type?} + Allowlist -->|yes| Domain[bounded decode to InterfoldEvent] + Allowlist -->|no| Reject[reject input] + Domain --> Handle[BusHandle remote publish] + Bus --> DocumentPublisher[DocumentPublisher] + DocumentPublisher --> Command[NetCommand channel] + Command --> Swarm + SyncManager --> EventStore[(EventStore query)] + SyncManager --> Budget[one startup budget: 512 pages / 50k events / 128 MiB / 5 min] + Budget --> Direct[versioned direct request/response] + Signed --> Notice[DHT document notification] + Notice --> Fetch[content-addressed DHT fetch] + Fetch --> MetaCheck{E3, kind, and party filter match payload?} + MetaCheck -->|yes| Handle + MetaCheck -->|no| Reject + Handle --> Bus[durable event pipeline] +``` + +The network interface owns the QUIC swarm, signed gossipsub topic, Kademlia store, and +transport channels. Gossipsub and direct-request/DHT decoding have explicit byte +limits. Translation actors accept only the protocol event allowlist before publishing +remote events, and their broadcast-to-actor ingress loops await mailbox acceptance and +stop when the destination actor closes. Startup buffering is bounded by both event +count and estimated bytes and fails readiness on overflow or broadcast lag. Historical +direct sync requires advancing cursors and enforces one cumulative page, event, byte, +and time budget across all aggregate fetches and recovery retries in a startup attempt. + +The gossiped `DocumentMeta` is independent of the DHT content hash, so +`EventConversionService::validate_received` decodes the fetched payload and binds the +metadata E3 identifier, `TrBFV` kind, and party-filter shape to that payload before a +`DocumentReceived` event is persisted. Transport and gossipsub identities authenticate +the sending peer; they do not by themselves prove that a peer is an authorized member +of a particular E3 committee. Topic/chain isolation, wire negotiation beyond the +current `0.0.1` identifiers, durable peer reputation, and end-to-end application +authorization remain separate protocol-hardening work. + +## E3 lifecycle + +```mermaid +stateDiagram-v2 + [*] --> None + None --> Requested: E3Requested + Requested --> CommitteeFinalized: CommitteeFinalized / CommitteePublished + CommitteeFinalized --> KeyPublished: PublicKeyAggregated + KeyPublished --> CiphertextReady: CiphertextOutputPublished + CiphertextReady --> Complete: PlaintextAggregated / PlaintextOutputPublished + Requested --> Failed: E3Failed + CommitteeFinalized --> Failed: E3Failed + KeyPublished --> Failed: E3Failed + CiphertextReady --> Failed: E3Failed + Complete --> [*] + Failed --> [*] +``` + +`E3LifecycleService` enforces monotonic progress and freezes terminal states. +`E3Router` creates and tears down per-request actor contexts. Duplicate and late +terminal observations are classified before forwarding; side effects are enabled +only after recovery. The diagram shows the normal progression: the lifecycle observer +also accepts a forward jump to a later stage, while reporting a lower-stage observation +as a regression without changing its tracked stage. + +## Committee, DKG, aggregation, and decryption + +```mermaid +sequenceDiagram + participant Chain as Chain events + participant S as Sortition + participant R as E3Router / context + participant K as ThresholdKeyshare + participant Z as ZK request + verification + participant P as PublicKeyAggregator + participant T as ThresholdPlaintextAggregator + participant W as Contract writers + + Chain->>S: E3Requested / tickets / committee finalization + S->>R: CiphernodeSelected + finalized committee + R->>K: create request-local DKG state + K->>Z: C1-C4 proof work + K->>Z: recipient bundle C2a, C2b, C3a x L_THRESHOLD, C3b x L_THRESHOLD + Z-->>K: canonical verified party results + K->>P: keyshare + proof per canonical party slot + P->>Z: folded/recursive aggregation proof work + P->>W: aggregated public key + Chain->>K: ciphertext outputs + K->>Z: C6 decryption-share proofs + K->>T: share + proof per output and party + T->>Z: C7 aggregation proof + T->>W: plaintext output +``` + +Committee order is the on-chain `topNodes` order; a party ID is an index into that +ordered committee. The Rust proof boundary validates canonical committee dimensions, +unique party slots, signer-to-slot binding, phase-specific proof multiplicity, and +one share/proof per ciphertext output. Circuit semantics are deliberately outside +this refactor's modification scope. + +Each recipient-scoped threshold-share bundle has one C2a secret-key share-computation +proof, one C2b smudging-noise share-computation proof, then every C3a proof, then every +C3b proof. C3 multiplicity follows rows of the threshold-parameter Shamir secret, not +the number of CRT moduli in the DKG encryption parameters: + +| Parameter pair | `L_THRESHOLD` | Recipient bundle | +| --- | ---: | --- | +| Insecure 512 | 2 | C2a x 1, C2b x 1, C3a x 2, C3b x 2 | +| Secure 8192 | 3 | C2a x 1, C2b x 1, C3a x 3, C3b x 3 | + +`ThresholdKeyshare` dispatches verification with the DKG/share-encryption preset. The +shape validator therefore normalizes a DKG preset to its threshold counterpart before +reading `num_moduli`; a threshold preset remains unchanged. This matches proof +generation, which creates one C3 request for each threshold Shamir row even though the +row is encrypted and proven with the paired DKG BFV parameters. The invariant is +independent of committee size for one recipient; full sender fanout has +`(N - 1) * L_THRESHOLD` C3a proofs and the same number of C3b proofs because the sender +does not encrypt its own slot. + +The current TrBFV implementation creates exactly one smudging-noise share set (`Z = 1`). +The general C3b multiplicity would be `Z * L_THRESHOLD` per recipient. Supporting +multiple ESI/smudging-noise sets requires coordinated producer, validator, NodeFold, +wire, and circuit work; the current validator must not silently infer that extension. + +## Failure, accusation, slashing, expulsion, and timeout + +```mermaid +flowchart TD + Invalid[invalid proof / commitment / missing work] --> Evidence[typed failure evidence] + Evidence --> Accuse[AccusationManager] + Accuse --> Votes[authenticated accusation votes] + Votes --> Quorum{honest threshold reached?} + Quorum -->|no| Wait[wait until deadline] + Quorum -->|yes| Decision[AccusationQuorumReached] + Wait --> Timeout[E3 timeout / formation failure] + Decision --> Writer[SlashingManagerSolWriter] + Writer --> Eligible{right chain, slashable outcome, and ranked submitter?} + Eligible -->|no| Ignore[ignore locally] + Eligible -->|yes| Effects{EffectsEnabled?} + Effects -->|no| Deferred[in-memory deferred intents] + Deferred -->|startup reconciliation complete| Submit + Effects -->|yes| Gate{semantic intent already deferred, in flight, or complete?} + Gate -->|yes| Ignore[coalesce duplicate] + Gate -->|no| Submit[submit now or after rank delay] + Submit --> Outcome{transaction outcome} + Outcome -->|success or classified benign result| Complete[retain completed key] + Outcome -->|retryable failure| Retry[clear in-flight key] + Retry --> Gate + Complete --> Chain[on-chain slash / expulsion] + Chain --> Registry[registry and committee observations] + Registry --> Lifecycle[E3 lifecycle / cleanup] + Timeout --> Lifecycle + + classDef residual fill:#fff1f0,stroke:#cf222e,color:#82071e + class Deferred residual +``` + +Vote quorum uses the honest threshold rather than the total committee size. +Cryptographic verification failures must be structurally attributable to a canonical +party before they become slashing evidence. Replayed `AccusationQuorumReached` events +are held until `EffectsEnabled`, then coalesced by the contract's semantic replay +domain across deferred, in-flight, and completed submissions. Retryable submission +failures release their key; successful or known-benign terminal results retain it. + +The gate is deliberately described as in-memory: there is no durable external-effect +outbox or persisted transaction intent. A crash after snapshot advancement but before +receipt classification can therefore lose the local redrive state, and a crash after +submission can require on-chain reconciliation to distinguish landed from missing +work. Closing that gap requires a durable intent/result state machine and contract +preflight, not another process-local set. + +## Program-server trust boundary + +```mermaid +flowchart LR + Caller[development compute caller] --> Json[JSON body limit: 10 MiB] + Json --> Callback{HTTP or HTTPS callback; no credentials or fragment?} + Callback -->|no| BadRequest[400] + Callback -->|yes| Capacity{job semaphore available?} + Capacity -->|no| Busy[429] + Capacity -->|yes| Compute[spawn FHE computation] + Compute --> Result[completed or failed payload] + Result --> Client[5 s connect / 30 s total; redirects disabled] + Client --> Allowed[caller-supplied callback URL] + + classDef residual fill:#fff1f0,stroke:#cf222e,color:#82071e + class Compute residual +``` + +`E3ProgramServerBuilder::build` fails only when the concurrency limit is zero. The +default limit is one job. The development/test endpoint does not authenticate callers +or allowlist callback destinations. The callback URL comes from the request body and +may target any HTTP(S) origin; URL credentials, fragments, and non-HTTP schemes are +rejected, and localhost rewriting changes only the exact host. The webhook client has +bounded connection/total time and does not follow redirects. Logs record result sizes +and response status rather than response bodies. + +The CRISP and default-template coordination callers do not log compute payloads. This +server is development tooling rather than an authenticated production compute boundary +and must not be exposed across trust boundaries. + +Accepted jobs are ordinary detached Tokio tasks. The semaphore bounds concurrent work, +but those tasks are not registered with an application shutdown token or join set, and +there is no durable job queue. A server stop can therefore cancel work or callback +delivery without a recoverable job record. The current health endpoint also reports +process availability, not dependency readiness or job-drain state. + +## Durable and in-memory ownership + +```mermaid +flowchart LR + subgraph Durable[Durable or externally authoritative] + Chain[(canonical EVM chain)] + Logs[(per-aggregate append-only event logs)] + Sled[(Sled repositories and snapshot cursors)] + Index[(derived SequenceIndex)] + Identity[(encrypted libp2p keypair repository)] + end + + subgraph Recovery[Startup reconstruction] + Reconcile[bounded log-to-index reconciliation] + Replay[post-snapshot event replay] + History[EVM and network historical reconciliation] + Unlock[decrypt identity] + end + + subgraph Memory[Process-local ownership] + Pipeline[HLC, admission, sequencer, and EventBus dedup] + Contexts[E3Router contexts and per-E3 protocol actors] + Network[swarm, peer state, buffers, and document interests] + Effects[nonce mutexes and slash-submission gate] + Accusations[in-flight accusation votes and timers] + Jobs[task pools and detached program-server jobs] + end + + Logs --> Reconcile --> Index + Logs --> Replay + Sled --> Replay + Replay --> Pipeline + Replay --> Contexts + Replay --> Effects + Chain --> History --> Contexts + Identity --> Unlock --> Network + Contexts --> Accusations + + classDef durable fill:#e6ffed,stroke:#238636,color:#116329 + classDef ephemeral fill:#fff1f0,stroke:#cf222e,color:#82071e + class Chain,Logs,Sled,Index,Identity durable + class Effects,Accusations,Jobs ephemeral +``` + +| State | Authoritative owner | Reconstructed from | +| --- | --- | --- | +| Protocol event history | Per-aggregate append-only event logs | Direct log scan | +| Aggregate snapshots and repositories | Sled-backed `Repositories` | Event replay after snapshot cursors | +| Timestamp index | `SequenceIndex` | Reconciled from event log on startup | +| Chain sync cursor | Aggregate snapshot metadata | Configured-confirmation EVM backfill | +| Network document history | Event log plus network repository | Historical net sync | +| E3 actor contexts | `E3Router` in memory | Durable replay and canonical chain observations | +| Request-local DKG/aggregation state | Per-E3 actors plus repositories | Snapshots, replay, and `EffectsEnabled` redrive | +| C0/share proof-verification context | Finalized-committee and ciphernode-selector repositories plus global verifier memory | Canonical slots and E3 preset/threshold metadata load before ZK actor startup, then lifecycle events maintain or clear them | +| HLC, EventBus dedup, and admission state | Event pipeline actors in memory | Maximum snapshot/replay HLC; a fresh bounded dedup window is populated by replay and live events | +| Network peer/buffer/interest state | libp2p and network actors in memory | Fresh peer dialing; document interest returns only when selection observations are replayed or redriven | +| Slash-submission replay gate | `SlashingManagerSolWriter` process memory | Rebuilt from replay; not a durable outbox | +| Pending transaction nonce allocation | Per-chain writer mutex in memory | Provider pending nonce on restart | +| In-flight accusation votes and timers | Per-E3 accusation actor memory | No complete durable reconstruction; only events inside the replay window may be observed again | +| libp2p identity | Encrypted keypair repository | Decrypt at startup | +| Program-server job permits/tasks | Tokio semaphore and detached tasks | Not reconstructed after process exit | + +No actor-local mutable cache is treated as durable merely because the actor survives +for the process lifetime. + +## Shutdown, restart, resync, and cancellation + +```mermaid +sequenceDiagram + participant CLI as signal loop / runtime + participant Bus as Event pipeline + participant Actors as protocol subscribers + participant Snap as SnapshotBuffer + participant Store as DataStore + + CLI->>Bus: close admission and persist Shutdown behind prior publishers + Bus->>Actors: ordered Shutdown, await acknowledgements + CLI->>Bus: flush sequencer, router, and event logs + CLI->>Snap: flush pending snapshot batches + CLI->>Store: flush Sled and close store actor + CLI->>CLI: flush log collector and return success or error +``` + +The whole barrier is time-bounded. Failure to drain or flush is returned to the CLI +and produces a non-zero exit. On restart, the process fence prevents two local +writers from sharing one database. Schema preflight rejects unsupported upgrades or +downgrades. `interfold node validate` provides offline integrity and loose-end +diagnostics; destructive automatic repair is not performed. + +The implemented restart and operator-controlled recovery boundary is: + +```mermaid +flowchart TD + Incident[unclean exit, corruption warning, or unsupported schema] --> Stop[stop the node and preserve its data] + Stop --> Validate[run interfold node validate offline] + Validate --> Decision{event log and schema usable?} + Decision -->|yes| Restart[normal node start] + Restart --> Preflight[schema preflight and bounded index reconciliation] + Preflight --> Replay[local snapshot plus event-log replay] + Replay --> History[confirmed EVM backfill plus bounded network sync] + History --> Available{required history available within hard budgets?} + Available -->|yes| Live[live mode after EffectsEnabled and SyncEnded fences] + Available -->|no| Blocked[remain stopped; no built-in full-resync override] + + Decision -->|no| Backup{verified compatible backup available?} + Backup -->|yes| Restore[restore with external filesystem tooling] + Restore --> Restart + Backup -->|no| Reset[explicit operator-controlled destructive data reset] + Reset --> Empty[empty-store start; reconstruct only from still-available chain and peer history] + Empty --> Restart + + classDef residual fill:#fff1f0,stroke:#cf222e,color:#82071e + class Blocked,Reset,Empty residual +``` + +There is no in-process repair, event-log rollback, backup/restore command, or dedicated +full-resync command in the Rust crates. Backup restore is an offline filesystem +operation. A destructive reset removes the local event log—the node's source of +truth—and can reconstruct only observations still available from configured EVM ranges +and peers. One historical network startup attempt, including all aggregates and +retries, is capped at 512 pages, 50,000 events, 128 MiB, and five minutes, with no +operator override in the current +implementation. Exceeding a budget or discovering unavailable history is therefore a +startup blocker, not a signal to silently skip data. Unsupported schema state likewise +requires a compatible binary, a verified backup, or an explicit reset; no automatic +migration is implemented. + +The multi-process SWARM supervisor has a separate child-process lifecycle: + +```mermaid +sequenceDiagram + participant API as supervisor API / SIGTERM + participant PM as ProcessManager + participant Child as managed child + participant Output as stdout/stderr forwarding tasks + + API->>PM: stop, stop_all, terminate, or partial-start cleanup + PM->>PM: remove process records without holding map lock across waits + PM->>Child: SIGTERM + alt child exits within 30 seconds + Child-->>PM: exit status + else grace period expires + PM->>Child: SIGKILL + Child-->>PM: forced exit + end + PM->>Output: drain for up to 5 seconds, then abort task + PM-->>API: success or first cleanup error + opt last Child handle is dropped on an error path + PM->>Child: kill-on-drop containment + end +``` + +An exited child is reported as `Exited { code }`, not `Started`, and can be started +again. A failure partway through `start_all` terminates children that already started. +Supervisor termination exits non-zero if child cleanup fails. Spawned handles use +`kill_on_drop`, so removing a process record before a failed termination step cannot +orphan the managed child. This is process +supervision only: it does not persist a desired-state/restart policy, and unexpected +child exit is observed through status rather than automatically restarted. + +## Error and cancellation propagation + +```mermaid +flowchart TD + Startup[configuration, schema, replay, EVM, or net startup error] --> StartResult[builder / entrypoint Result] + StartResult --> CLIExit[CLI returns failure] + + Handler[recoverable actor or adapter error] --> Trap[trap / trap_fut or BusHandle::err] + Trap --> ErrorEvent[typed InterfoldError event] + ErrorEvent --> Durable[normal durable event pipeline] + Durable --> Observers[logs, collectors, and interested actors] + + Mailbox[awaited actor mailbox closes] --> Producer[bridge or replay producer receives error] + Producer --> StopLoop[stop ingress loop or fail startup] + Buffer[network startup overflow or broadcast lag] --> Readiness[fail readiness] + Readiness --> StartResult + + StoreFailure[event-log or index write failure] --> StoreDeath[affected EventStore panics before dispatch] + StoreDeath --> PipelineStall[sequencer loses its acknowledgement path] + StoreDeath --> ProcessAlive[process may remain alive because unwind stops at the actor task] + + Signal[SIGINT / SIGTERM] --> Close[close BusHandle admission] + Close --> Shutdown[acknowledged Shutdown fanout] + Shutdown --> Flush[event log and snapshot/store flush] + Flush --> Deadline{shutdown deadline met?} + Deadline -->|yes| Clean[successful exit] + Deadline -->|no or flush error| CLIExit + + Shutdown --> Cancel[oneshot shutdown sender] + Cancel --> Reader[EVM reader retry loop exits] + + Detached[detached protocol or program-server tasks] -. not uniformly joined .-> ProcessExit[process exit] + + classDef residual fill:#fff1f0,stroke:#cf222e,color:#82071e + class PipelineStall,ProcessAlive,Detached residual +``` + +Startup barriers propagate errors to the caller and fail closed where continuing would +drop historical or live input. Awaited network bridges stop when their destination +actor closes. Recoverable handler failures are generally converted into durable +`InterfoldError` events by the existing `trap` helpers; a logged error is not itself a +supervision restart. EventStore write failures preserve safety by panicking before +dispatch, but the default unwind build does not turn a spawned Actix actor panic into a +guaranteed process exit. The store actor can die while the process remains present and +the sequencer stalls; current code has no supervisor that converts that condition into +a deterministic restart. Shutdown closes event admission, waits for admitted +publishers, persists and fans out `Shutdown`, flushes the durable pipeline, then drains +snapshot batches and the backing store under one deadline. Any failed shutdown stage +reaches the CLI and causes an unsuccessful exit. + +Cancellation ownership is not uniform across the workspace. EVM reader loops have an +explicit oneshot shutdown signal, the EventBus shutdown event stops many actors, and +the outer deadline prevents an indefinite drain. Detached tasks without a join handle +or cancellation token remain a residual: process exit is their final cancellation +boundary, so they cannot all prove completion or persist recovery intent. + +## Subsystem contracts + +| Subsystem | Responsibility and I/O | Owned state and dependencies | Invariant and failure behavior | Extension boundary / must not own | +| --- | --- | --- | --- | --- | +| `e3-events` | Admit, timestamp, persist, deduplicate, and fan out typed events. | HLC factory, subscriber registry, sequencer, event stores, and snapshot bridge; depends on Actix and protocol payload types. | Event log append precedes live dispatch; startup index reads are paged; query responses and shutdown/replay/recovery-phase barriers are acknowledged. Storage or mailbox failures reach a caller where the path is awaited, but live append/response `do_send` edges remain. | Event/subscription APIs; must not own request-specific protocol policy. | +| `e3-data` | Serve typed repository reads/writes and append-only event-log/index records. | Sled/in-memory stores, log handles, batch writes, and flush failure state. | Acknowledged sync/batch writes flush before success; decode corruption and recorded write failures fail closed. | Repository/store factories; must not decide committees, proofs, or lifecycle transitions. | +| `e3-sync` | Reconstruct actor state and reconcile EVM/network history before live mode. | Startup plan, disk-backed local replay runs, and bounded reconciled-history vectors; depends on repositories, EventBus, EVM, and net adapters. | Schema is checked before state-writing actors; HLC includes post-snapshot history; replay is global-HLC ordered with acknowledged subscriber acceptance; Effects/history/SyncEnded phases are downstream-acknowledged; history gaps or bounded-net-sync failure abort startup. | Historical collectors/planners; must not submit live transactions. | +| `e3-net` | Translate bounded libp2p traffic and serve gossip, DHT, and historical sync. | Swarm, Kademlia records, peer/transport status, channels, startup buffer, and document interests. | Signed gossip is type-allowlisted; decodes, startup backlog, and sync fetches are bounded; metadata must match DHT payload. Errors fail readiness or stop the affected ingress loop. | `NetInterface` and pure translation services; must not own E3 transitions or infer committee authority from PeerId alone. | +| `e3-evm` | Read chain history under the configured confirmation policy and submit typed contract transactions. | Per-chain gateways, provider handles, chain buffers, nonce mutexes, and slash replay gate. | Malformed logs and reverted receipts fail; nonce allocation is serialized in-process; no durable transaction outbox or full reorg rollback exists. | Provider/contract helpers; must not own off-chain proof policy. | +| `e3-request` | Route E3-scoped events and enforce lifecycle progress. | `E3Router`, lifecycle state, typed `(E3, recipient)` buffers, and request actor contexts; depends on event and protocol actor APIs. | Legal progress is monotonic; buffered history precedes the recipient-creating event; terminal teardown purges absent-role buffers. Active buffer size and child `do_send` remain residual risks. | Domain lifecycle/routing functions; must not implement storage, network framing, or contract decoding. | +| `e3-sortition` | Track registry/tickets and derive canonical selection/committee observations. | Node registry, ticket state, selector backend, and chain-derived committee state. | On-chain ordering is authoritative; terminal cleanup releases local participation state and removes durable finalized-committee and pending-expulsion records. | Sortition backend; must not construct cryptographic proofs. | +| `e3-keyshare` | Coordinate request-local DKG, shares, and decryption work. | Threshold keyshare actor state and repositories; depends on FHE/ZK services and the event bus. | Party IDs index the canonical committee; each recipient gets C2a/C2b singletons and C3a/C3b per threshold Shamir row; resumable determined outputs redrive only after `EffectsEnabled`. | Cryptographic backend/task pool; must not own transport frames or ABI decoding. | +| `e3-zk-prover` | Build and verify typed proof jobs/statements. | Backend job state, circuit registry, verification outcomes, and durable-seeded in-memory committee/preset caches. | Statement shapes, canonical committee dimensions, signer/slot binding, and proof multiplicity are checked before acceptance; DKG presets normalize to their threshold counterpart when deriving C3 row counts. Finalized slots plus C0 preset/threshold context load before replay so snapshot cursors cannot erase signer authority or artifact selection on restart. | ZK backend and registry; must not add committee policy absent from the proof statement. | +| `e3-aggregator` | Aggregate canonical verified public-key/plaintext shares. | Explicit per-E3 aggregation state machines and repositories. | One signer-bound share/proof occupies each canonical party slot and output multiplicity is exact; invalid or duplicate contributions are rejected. | Pure aggregation states and proof backend; must not own EVM transaction policy. | +| `e3-slashing` | Attribute proof failures, collect authenticated votes, and emit quorum outcomes. | Accusation/evidence/vote state; depends on committee data, verification, and events. | Honest threshold decides quorum and only structurally attributable failures become evidence. | Voting/evidence domain modules; must not generate proofs or assign ambiguous blame. | +| `e3-program-server` | Serve bounded development compute requests and deliver results to caller-supplied HTTP(S) callbacks. | Runner closure, callback client, and job semaphore. | Zero job capacity fails build; overload returns 429; callbacks reject unsafe URL forms and use bounded delivery timeouts. The test endpoint does not authenticate callers, does not allowlist callback targets, and must not be exposed as a production service. Detached tasks are not recoverable. | Runner callback; must not become durable protocol state or be treated as a production trust boundary. | +| `e3-ciphernode-builder` | Construct concrete stores, adapters, actor extensions, and startup barriers. | Composition handles and validated configuration, not protocol state. | Required components and startup readiness must succeed before returning a handle. | Concrete factories/extensions; must not accumulate protocol policy or durable business state. | +| `e3-entrypoint` / SWARM supervisor | Load/decrypt node configuration and manage child processes. | Process map, kill-on-drop child handles, and output-forwarding tasks. | Partial startup is cleaned up; status distinguishes exited children; stop is SIGTERM-first and time-bounded; a dropped final handle cannot orphan its child. | Command composition; must not silently restart failed protocol work or own node domain state. | + +Extension points should be narrow concrete boundaries with an active consumer: +repository factories, network interfaces, ZK backends, sortition backends, clocks, +and task pools. New one-method traits are not introduced solely to create layers. diff --git a/agent/RULES.md b/agent/RULES.md index a07203af05..b879db61ce 100644 --- a/agent/RULES.md +++ b/agent/RULES.md @@ -12,6 +12,9 @@ its content. - `tests/` — Integration tests - `agent/` — LLM context documentation +For Rust work, read both `agent/ARCHITECTURE.md` (contribution rules) and +`agent/CRATES_ARCHITECTURE.md` (the implemented runtime, persistence, and protocol topology). + ## Build configuration: preset and committee Two orthogonal axes pick what gets compiled into `circuits/bin/`: diff --git a/agent/flow-trace/00_INDEX.md b/agent/flow-trace/00_INDEX.md index b1f56b8b52..3607eb30d0 100644 --- a/agent/flow-trace/00_INDEX.md +++ b/agent/flow-trace/00_INDEX.md @@ -184,15 +184,17 @@ _Found during source-code cross-referencing of these trace documents._ ### Protocol Design Concerns -| # | Concern | Severity | Detail | -| --- | ------------------------------------------ | -------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | **Deregister-before-slash race** | Accepted | SlashingManager Lane B (evidence+appeal) has a window during which the operator can deregister and claim their exit. If they do, the slash executes against 0 funds. The contract comments acknowledge this as an accepted tradeoff for the appeal window design. | -| 2 | **Committee publication decentralized** | Resolved | `publishCommittee()` is permissionless. Off-chain role selection chooses the active aggregator, while on-chain C5 proof verification and the single-publish guard prevent invalid or duplicate committee publication. | -| 3 | **`gracePeriod` is dead code** | Medium | `gracePeriod` is stored and validated during config updates but never actually used in any timeout check. Either the deadlines already bake in sufficient buffer, or this is a missing feature. | -| 4 | **`activate` CLI command is misleading** | Low | Named "activate" but actually calls "register" — will fail for already-registered operators. There's no standalone way to trigger re-evaluation of active status; instead, `_updateOperatorStatus()` runs automatically inside `addTicketBalance()`, `bondLicense()`, etc. | -| 5 | **Active-job load balancing bug fixed** | Info | The Rust `NodeStateStore.available_tickets()` subtracts `active_jobs` from total tickets, reducing the chance of busy nodes being selected for new E3s. Previously, the `Sortition` actor's `Handler` was missing match arms for `E3Failed` and `E3StageChanged`, causing these events to fall to the default `_ => ()` — the typed handlers for decrementing jobs were dead code. This has been fixed: E3Failed and E3StageChanged are now routed to their handlers, and `finalized_committees` is cleaned up in `decrement_jobs_for_e3` to prevent unbounded memory growth. | -| 6 | **Committee member expulsion** | Info | `SlashingManager` can call `expelCommitteeMember()` mid-DKG. The `Sortition` actor enriches the raw `CommitteeMemberExpelled` event with the expelled member's `party_id` (resolved from its stored `Committee` list) and re-publishes it. `ThresholdKeyshare` then uses the enriched `party_id` to update its collectors, potentially completing DKG with fewer parties. `ThresholdKeyshare` itself does not hold committee state. | -| 7 | **ProofRequestActor failure bridge fixed** | Info | `ProofRequestActor` no longer leaves proof publication suppressed under log-only "will not be published" exits. `ComputeRequestError` and local proof-signing failures for DKG-path proofs (`C0` through `C5`) now emit `E3Failed { failed_at_stage: CommitteeFinalized, reason: DKGInvalidShares }`, while decryption-path proofs (`C6` and `C7`) emit `E3Failed { failed_at_stage: CiphertextReady, reason: DecryptionInvalidShares }`. | -| 8 | **Settlement receipts isolated** | Resolved | Durable EVM receipts such as `RewardCredited` and `RewardClaimed` are global audit/projection facts. The E3 router no longer sends them into a completed per-E3 context, so reward fan-out cannot reopen a finished E3 or produce false `AlreadyCompleted` failures. | -| 9 | **Replay-safe compute effects** | Resolved | `ComputeEffectGate` subscribes before EventStore replay, buffers `ComputeRequest`s while effects are disabled, deduplicates equivalent requests, prefers the newest hydrated retry, cancels terminal E3 work, and releases pending effects only after `EffectsEnabled`. This closes the mid-E3 compute-loss window without changing durable event order. | +| # | Concern | Severity | Detail | +| --- | --------------------------------------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | **Deregister-before-slash race** | Accepted | SlashingManager Lane B (evidence+appeal) has a window during which the operator can deregister and claim their exit. If they do, the slash executes against 0 funds. The contract comments acknowledge this as an accepted tradeoff for the appeal window design. | +| 2 | **Committee publication decentralized** | Resolved | `publishCommittee()` is permissionless. Off-chain role selection chooses the active aggregator, while on-chain C5 proof verification and the single-publish guard prevent invalid or duplicate committee publication. | +| 3 | **`gracePeriod` is dead code** | Medium | `gracePeriod` is stored and validated during config updates but never actually used in any timeout check. Either the deadlines already bake in sufficient buffer, or this is a missing feature. | +| 4 | **`activate` CLI command is misleading** | Low | Named "activate" but actually calls "register" — will fail for already-registered operators. There's no standalone way to trigger re-evaluation of active status; instead, `_updateOperatorStatus()` runs automatically inside `addTicketBalance()`, `bondLicense()`, etc. | +| 5 | **Active-job load balancing bug fixed** | Info | The Rust `NodeStateStore.available_tickets()` subtracts `active_jobs` from total tickets, reducing the chance of busy nodes being selected for new E3s. Previously, the `Sortition` actor's `Handler` was missing match arms for `E3Failed` and `E3StageChanged`, causing these events to fall to the default `_ => ()` — the typed handlers for decrementing jobs were dead code. This has been fixed: E3Failed and E3StageChanged are now routed to their handlers, and `finalized_committees` is cleaned up in `decrement_jobs_for_e3` to prevent unbounded memory growth. | +| 6 | **Committee member expulsion** | Info | `SlashingManager` can call `expelCommitteeMember()` mid-DKG. The `Sortition` actor enriches the raw `CommitteeMemberExpelled` event with the expelled member's `party_id` (resolved from its stored `Committee` list) and re-publishes it. `ThresholdKeyshare` then uses the enriched `party_id` to update its collectors, potentially completing DKG with fewer parties. `ThresholdKeyshare` itself does not hold committee state. | +| 7 | **ProofRequestActor failure bridge fixed** | Info | `ProofRequestActor` no longer leaves proof publication suppressed under log-only "will not be published" exits. `ComputeRequestError` and local proof-signing failures for DKG-path proofs (`C0` through `C5`) now emit `E3Failed { failed_at_stage: CommitteeFinalized, reason: DKGInvalidShares }`, while decryption-path proofs (`C6` and `C7`) emit `E3Failed { failed_at_stage: CiphertextReady, reason: DecryptionInvalidShares }`. | +| 8 | **Settlement receipts isolated** | Resolved | Durable EVM receipts such as `RewardCredited` and `RewardClaimed` are global audit/projection facts. The E3 router no longer sends them into a completed per-E3 context, so reward fan-out cannot reopen a finished E3 or produce false `AlreadyCompleted` failures. | +| 9 | **Replay-safe compute effects** | Resolved | `ComputeEffectGate` subscribes before EventStore replay, buffers `ComputeRequest`s while effects are disabled, deduplicates equivalent requests, prefers the newest hydrated retry, cancels terminal E3 work, and releases pending effects only after `EffectsEnabled`. This closes the mid-E3 compute-loss window without changing durable event order. | | 10 | **Restarted active aggregator decryption recovery** | Resolved | A node killed after key publication but before ciphertext publication could miss the one-shot `AggregatorChanged` role event before its plaintext buffer existed, lack recovered committee dependencies when ciphertext arrived, or rehydrate without the per-E3 `CommitmentConsistencyChecker` needed to answer C6 verification gates. `ThresholdPlaintextAggregatorExtension` records aggregator role, remembers pending ciphertext until committee facts arrive, recovers full committee data from public-key state/replayed public events, recovers the honest subset from public-key or threshold-keyshare state, `CommitmentConsistencyCheckerExtension` recreates its per-E3 checker from recovered meta, and restart recovery never blocks the router on synchronous store reads. | +| 11 | **Large EventStore replay startup failure** | Resolved | Restart replay now awaits each EventBus handler result before submitting the next persisted event. A backlog larger than the 2,560-entry EventBus mailbox no longer fails startup with `SendError::Full`; progress is logged every 10,000 EventBus-handled events. The EventBus still fans out with unacknowledged `do_send`, and replay still materializes the full range in memory, so end-to-end queue and memory bounds remain open. | +| 12 | **Canonical DKG H/N aggregation shape** | Resolved | The Rust DKG aggregation path no longer asserts that honest proof count `H` equals full committee size `N`. Every supported preset has `H < N`; validation now requires exactly `H` NodeFold proofs with unique in-range party IDs and exactly `N` ordered committee addresses, matching the producer and compiled Noir witness. The removed assertion affected debug/test binaries; circuit semantics and public inputs are unchanged. | diff --git a/agent/flow-trace/04_DKG_AND_COMPUTATION.md b/agent/flow-trace/04_DKG_AND_COMPUTATION.md index c1d582a5f6..20b83276df 100644 --- a/agent/flow-trace/04_DKG_AND_COMPUTATION.md +++ b/agent/flow-trace/04_DKG_AND_COMPUTATION.md @@ -606,6 +606,9 @@ ThresholdKeyshare receives AllThresholdSharesCollected │ │ ├─ Dispatches ComputeRequest::zk(ZkRequest::DkgAggregation { │ │ │ node_fold_proofs, c5_proof, party_ids, params_preset │ │ │ }) +│ │ │ → exactly H NodeFold proofs and H unique party ids +│ │ │ → exactly N ordered committee addresses (`topNodes`), where canonical H < N +│ │ │ → Rust validates both dimensions before invoking the compiled circuit │ │ ├─ Tracks the in-flight correlation id │ │ ├─ ComputeRequestError now emits │ │ │ E3Failed { failed_at_stage: CommitteeFinalized, reason: DKGInvalidShares } diff --git a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md index 07322c5b32..304b6c5e7b 100644 --- a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md +++ b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md @@ -183,6 +183,7 @@ publishPlaintextOutput() succeeds ├─ Sortition: decrements activeJobs for each committee member │ → Node becomes available for future E3s │ → Removes e3_id from node_state.e3_committees map + │ → Removes the durable finalized-committee and pending-expulsion records │ ├─ CiphernodeSelector: removes e3_id from e3_cache, committee, expelled set, │ and persisted aggregator designation for the E3 @@ -207,11 +208,12 @@ interfold start → running node │ ├─ Ctrl+C / SIGINT / SIGTERM │ -└─ listen_for_shutdown(): - ├─ Signals EventBus to stop - ├─ Awaits join_handle (main actor system) - ├─ Persists final state to Sled DB - └─ Clean exit +└─ graceful_shutdown(): + ├─ Persists Shutdown and waits for acknowledged EventBus fanout + ├─ Flushes the sequencer and event-store pipeline + ├─ Drains open snapshot batches, flushes the backing store, and closes it + ├─ Enforces a 30-second deadline and exits unsuccessfully on failure + └─ Flushes the optional operational JSON log collector On restart: ├─ Sync module replays: @@ -224,11 +226,18 @@ On restart: │ → ProofVerificationActor loads the same slots, plus BFV preset/threshold │ metadata from durable CiphernodeSelector state. Snapshotted │ CiphernodeSelected events are likewise not guaranteed to replay. +│ → Terminal lifecycle state prunes stale finalized-committee and +│ pending-expulsion records before actors start. │ 2. CiphernodeSelector emits persisted AggregatorChanged state before replay │ → ThresholdPlaintextAggregatorExtension records this role in the E3 context │ so a plaintext buffer created later by CiphertextOutputPublished starts │ with the correct active-aggregator flag │ 3. Replay EventStore events since last snapshot (effects still disabled) +│ → Read each aggregate in 1,024-event pages, sort bounded temporary runs, +│ and perform a bounded-fan-in global merge by HLC timestamp +│ → Each concurrent EventBus fanout is acknowledged before the next event; +│ an unavailable or blocked listener aborts recovery after a bounded wait +│ → Structured progress is emitted every 10,000 EventBus-handled events │ 4. Fetch historical EVM events from last known block │ 5. Historical libp2p sync retries failed aggregate fetches after reconnects │ and also on bounded retry intervals even without a new connection event @@ -242,6 +251,25 @@ On restart: └─ Node resumes from where it left off ``` +The shutdown barrier proves that the persisted `Shutdown` event reached its current subscribers, +the event pipeline flushed, open snapshot batches drained, and the backing store flushed within the +deadline. Detached work that is not owned by those barriers can still be cancelled by process exit; +operators must continue to follow the production shutdown precautions. + +The three long-lived libp2p `NetEvent` broadcast consumers (`NetEventTranslator`, +`DocumentPublisher`, and `NetSyncManager`) treat Tokio's `Lagged(n)` receive result as a recoverable +overload signal: they emit a bounded structured warning containing only the static consumer name and +skipped-event count, then continue from the oldest retained event. Only channel closure ends a +receive task. A lag can still drop the reported `n` events, but a single burst no longer permanently +disables gossip translation, document notifications, or historical-sync/readiness handling. + +EventStore replay uses a disk-backed external merge: per-aggregate pages are sorted into secure +temporary runs, then compacted and merged with bounded file-descriptor fan-in. Replay waits for +concurrent EventBus listener acceptance for each event. A listener that is unavailable or cannot +accept within the timeout fails recovery instead of being silently skipped. Snapshot routing still +contains asynchronous edges, so this does not claim that every downstream actor is synchronously +durable at each replay step. + ### Restart + Persist State Diagram ```mermaid @@ -304,30 +332,35 @@ settlement observations) remain in EventStore for auditing and operator projecti not deliver them to a completed per-E3 context because they report settlement; they do not resume protocol execution. +`CiphernodeSelector` also emits every persisted `AggregatorChanged` entry before EventStore replay. +If a prior snapshot failed to persist the selector's completion cleanup, the request router may log +that emission as unexpected for an already-completed E3. The router converts it to an +`InterfoldError`; it does not abort EventBus replay. Treat the warning as evidence of stale snapshot +state rather than suppressing it unconditionally. + For crashes after key publication but before ciphertext publication, the recovered active aggregator may not have a `ThresholdPlaintextAggregator` actor yet when the persisted `AggregatorChanged` event -is re-emitted. The plaintext extension records that role in the live E3 context, then seeds the later -`DecryptionshareCreatedBuffer` from it. Committee and honest-committee addresses are recovered +is re-emitted. The plaintext extension records that role in the live E3 context, then seeds the +later `DecryptionshareCreatedBuffer` from it. Committee and honest-committee addresses are recovered from completed public-key aggregation state, in-flight public-key aggregation state, or the persisted `ThresholdKeyshareState.honest_parties` set during async context hydration. Replayed `CommitteePublished` can also restore the full committee address dependency, but cannot infer the H-sized honest subset when `N > H`; that subset must come from `PublicKeyAggregated`, `PublicKeyAggregatorState::GeneratingC5Proof`, or threshold-keyshare state. The synchronous `on_event` path must not read actor-backed repositories directly, because blocking the router while -waiting for the store can freeze live gossip and make peers time out. -If `CiphertextOutputPublished` is replayed before those committee dependencies are ready, the -extension records the ciphertext in the E3 context and retries plaintext actor creation when -`PublicKeyAggregated` or `CommitteePublished` supplies the missing facts; the router's existing -recipient buffer then drains any ciphertext/decryption-share events into the newly-created -plaintext path. - -`ShareVerificationActor` gates C1/C6 proof verification behind -`CommitmentConsistencyCheckRequested` / `CommitmentConsistencyCheckComplete`. The per-E3 -`CommitmentConsistencyChecker` is therefore restart-critical even though it has no durable state of -its own: after context hydration, `CommitmentConsistencyCheckerExtension` recreates it from the -recovered `E3Meta` so restarted active aggregators can complete C6 verification. Without this -recipient, the restarted node can collect honest decryption shares and then wait forever for a -consistency-check response that no actor is subscribed to publish. +waiting for the store can freeze live gossip and make peers time out. If `CiphertextOutputPublished` +is replayed before those committee dependencies are ready, the extension records the ciphertext in +the E3 context and retries plaintext actor creation when `PublicKeyAggregated` or +`CommitteePublished` supplies the missing facts; the router's existing recipient buffer then drains +any ciphertext/decryption-share events into the newly-created plaintext path. + +`ShareVerificationActor` gates C1/C6 proof verification behind `CommitmentConsistencyCheckRequested` +/ `CommitmentConsistencyCheckComplete`. The per-E3 `CommitmentConsistencyChecker` is therefore +restart-critical even though it has no durable state of its own: after context hydration, +`CommitmentConsistencyCheckerExtension` recreates it from the recovered `E3Meta` so restarted active +aggregators can complete C6 verification. Without this recipient, the restarted node can collect +honest decryption shares and then wait forever for a consistency-check response that no actor is +subscribed to publish. The global `ShareVerificationActor` also requires the finalized committee's ordered party-slot map for signer ownership checks. It is seeded from `Repositories::finalized_committees` during builder @@ -348,15 +381,15 @@ finalized-committee repository and `CiphernodeSelectorState.e3_cache` before rep The node is choreographed — each subsystem reacts to bus events independently — so there is no single component that _drives_ the protocol. The `E3LifecycleCoordinator` (in `e3-request`) is an -**additive, durable observer** that gives the node a single source of truth for "what stage is each -E3 at?". It never emits protocol events and never drives subsystems; it only records stage and -supports restart-resume and shutdown awareness. +**additive persisted-stage observer** that gives the live node one projection of "what stage is each +E3 at?". It never emits protocol events and never drives subsystems; it records stage and supports +restart-resume and shutdown awareness subject to the asynchronous persistence caveats above. ``` E3LifecycleCoordinator::attach(bus, store) (wired in ciphernode_builder.build()) │ ├─ Loads persisted stage map from Repository(StoreKeys::e3_lifecycle()) -│ → on restart, every in-flight E3's last known stage is rehydrated +│ → on restart, every successfully persisted in-flight stage is rehydrated │ ├─ Subscribes to lifecycle-bearing events: │ E3Requested → Requested @@ -374,11 +407,11 @@ E3LifecycleCoordinator::attach(bus, store) (wired in ciphernode_builder.build( │ • Advance is MONOTONIC (forward-only by stage rank) │ • Out-of-order earlier-stage events are logged (Regressed) and ignored │ • Once Complete/Failed, the stage is frozen (Terminal) -│ On Advanced/Terminal the snapshot is persisted (set on Persistable) +│ On Advanced/Terminal, updates memory and enqueues a snapshot write │ └─ On Shutdown event: logs the set of still-active (non-terminal) E3s and their stages, - persists the final snapshot, then stops. + enqueues a final snapshot write, then stops without awaiting durability. ``` The coordinator is safe by construction during EventStore replay: observing a replayed lifecycle From 36438dbe2af193f91fe267f867af20e50a7abc10 Mon Sep 17 00:00:00 2001 From: Hamza Khalid <36852564+hmzakhalid@users.noreply.github.com> Date: Tue, 14 Jul 2026 21:35:31 +0500 Subject: [PATCH 55/62] refactor: organize ciphernode crates by capability [skip-line-limit] (#1702) --- agent/ACTOR_AUDIT.md | 65 + agent/ARCHITECTURE.md | 803 +++--- agent/CRATES_ARCHITECTURE.md | 80 +- agent/flow-trace/04_DKG_AND_COMPUTATION.md | 15 +- .../flow-trace/05_FAILURE_REFUND_SLASHING.md | 7 +- .../src/{actors/mod.rs => actors.rs} | 10 +- .../src/actors/publickey_aggregator.rs | 1961 -------------- .../actors/threshold_plaintext_aggregator.rs | 1416 ---------- .../aggregator/src/{domain => }/committee.rs | 0 .../actor.rs} | 142 +- .../src/committee_finalization/handlers.rs | 144 + .../src/{domain => }/committee_hash.rs | 0 crates/aggregator/src/domain.rs | 14 + crates/aggregator/src/domain/mod.rs | 15 - .../src/domain/publickey_aggregation.rs | 750 ------ .../domain/threshold_plaintext_aggregation.rs | 645 ----- .../aggregator/src/{domain => }/failover.rs | 0 crates/aggregator/src/lib.rs | 10 +- .../src/plaintext_aggregation/actor.rs | 196 ++ .../decryption_share_buffer.rs} | 0 .../src/plaintext_aggregation/effects/mod.rs | 13 + .../effects/prove_plaintext.rs | 269 ++ .../effects/publish_result.rs | 126 + .../effects/verify_decryption_shares.rs | 199 ++ .../src/plaintext_aggregation/handlers.rs | 291 ++ .../src/plaintext_aggregation/intents.rs | 51 + .../src/plaintext_aggregation/state.rs | 139 + .../plaintext_aggregation/tests/completion.rs | 68 + .../plaintext_aggregation/tests/failures.rs | 168 ++ .../src/plaintext_aggregation/tests/mod.rs | 181 ++ .../src/plaintext_aggregation/transitions.rs | 247 ++ .../src/plaintext_aggregation/workflow.rs | 46 + .../plaintext_aggregation/workflow_tests.rs | 191 ++ .../src/public_key_aggregation/actor.rs | 86 + .../effects/aggregate_dkg_proofs.rs | 242 ++ .../effects/aggregate_public_key.rs | 231 ++ .../effects/fold_node_proofs.rs | 204 ++ .../effects/handle_compute_results.rs | 254 ++ .../src/public_key_aggregation/effects/mod.rs | 28 + .../effects/publish_result.rs | 129 + .../effects/verify_key_proofs.rs | 290 ++ .../src/public_key_aggregation/handlers.rs | 212 ++ .../keyshare_buffer.rs} | 0 .../src/public_key_aggregation/state.rs | 133 + .../tests/attestations.rs | 99 + .../public_key_aggregation/tests/failures.rs | 114 + .../src/public_key_aggregation/tests/mod.rs | 173 ++ .../src/public_key_aggregation/transitions.rs | 256 ++ .../src/public_key_aggregation/validation.rs | 132 + .../src/public_key_aggregation/workflow.rs | 45 + .../public_key_aggregation/workflow_tests.rs | 214 ++ crates/aggregator/src/workflow.rs | 15 + crates/data/src/persistable.rs | 96 +- crates/evm/src/{actors/mod.rs => actors.rs} | 17 +- .../evm/src/actors/ciphernode_registry_sol.rs | 590 ---- crates/evm/src/actors/evm_chain_gateway.rs | 455 ---- crates/evm/src/actors/evm_read_interface.rs | 526 ---- crates/evm/src/actors/log_fetcher.rs | 672 ----- crates/evm/src/adapters.rs | 6 + .../actor.rs} | 0 .../events.rs} | 0 crates/evm/src/chain_gateway/actor.rs | 214 ++ crates/evm/src/chain_gateway/handlers.rs | 48 + .../state.rs} | 0 crates/evm/src/chain_gateway/tests.rs | 210 ++ .../src/{actors/evm_hub.rs => chain_hub.rs} | 0 crates/evm/src/chain_reader/actor.rs | 154 ++ .../src/{domain => chain_reader}/backoff.rs | 0 crates/evm/src/chain_reader/effects/mod.rs | 8 + .../chain_reader/effects/provider_recovery.rs | 117 + crates/evm/src/chain_reader/effects/stream.rs | 236 ++ crates/evm/src/chain_reader/handlers.rs | 47 + .../{domain => chain_reader}/log_timestamp.rs | 0 .../evm/src/{domain => chain_reader}/reorg.rs | 0 .../start_extractor.rs} | 0 crates/evm/src/ciphernode_registry/actor.rs | 121 + crates/evm/src/ciphernode_registry/effects.rs | 230 ++ .../events.rs} | 0 .../evm/src/ciphernode_registry/handlers.rs | 267 ++ crates/evm/src/{domain/mod.rs => domain.rs} | 17 +- .../evm_parser.rs => event_decoding/actor.rs} | 0 .../catalog.rs} | 0 .../error.rs} | 0 .../observation.rs} | 0 .../{actors/evm_router.rs => event_router.rs} | 0 .../actor.rs} | 0 .../workflow.rs} | 0 .../events.rs} | 0 .../reader.rs} | 0 crates/evm/src/interfold_writing/actor.rs | 91 + crates/evm/src/interfold_writing/effects.rs | 94 + .../handlers.rs} | 176 +- .../workflow.rs} | 0 crates/evm/src/lib.rs | 5 +- crates/evm/src/log_fetching/adapter.rs | 264 ++ crates/evm/src/log_fetching/tests/backfill.rs | 145 + crates/evm/src/log_fetching/tests/fetch.rs | 147 + crates/evm/src/log_fetching/tests/mod.rs | 139 + .../slashing_events.rs => slashing/events.rs} | 0 .../evidence.rs} | 0 .../reader.rs} | 0 crates/evm/src/slashing_writing/actor.rs | 104 + crates/evm/src/slashing_writing/effects.rs | 111 + .../handlers.rs} | 203 +- .../workflow.rs} | 0 .../keyshare/src/{actors/mod.rs => actors.rs} | 8 +- .../keyshare/src/actors/threshold_keyshare.rs | 2372 ----------------- .../keyshare/src/{domain/mod.rs => domain.rs} | 13 +- .../keyshare/src/threshold_keyshare/actor.rs | 231 ++ .../collect_decryption_keys.rs} | 0 .../collect_encryption_keys.rs} | 0 .../collect_threshold_shares.rs} | 0 .../decryption_key_collector.rs} | 0 .../derive_decryption_key.rs} | 0 .../effects/calculate_decryption_key.rs | 194 ++ .../effects/coordinate_collectors.rs | 246 ++ .../effects/create_decryption_share.rs | 270 ++ .../effects/generate_threshold_share.rs | 169 ++ .../effects/initialize_dkg.rs | 208 ++ .../src/threshold_keyshare/effects/mod.rs | 19 + .../effects/route_events.rs | 151 ++ .../effects/track_proofs.rs | 110 + .../effects/verify_decryption_key.rs | 174 ++ .../effects/verify_threshold_shares.rs | 284 ++ .../encryption_key_collector.rs | 0 .../generate_encryption_key.rs} | 0 .../generate_shares.rs} | 0 .../src/threshold_keyshare/handlers.rs | 244 ++ .../state.rs} | 184 +- .../src/threshold_keyshare/state_tests.rs | 186 ++ .../keyshare/src/threshold_keyshare/tests.rs | 173 ++ .../threshold_share_collector.rs | 0 .../timeout_policy.rs | 0 crates/net/src/actors.rs | 31 + crates/net/src/actors/document_publisher.rs | 1007 ------- crates/net/src/actors/net_event_buffer.rs | 330 --- crates/net/src/actors/net_sync_manager.rs | 1068 -------- crates/net/src/document_publishing/actor.rs | 157 ++ crates/net/src/document_publishing/effects.rs | 163 ++ .../net/src/document_publishing/handlers.rs | 103 + .../net/src/document_publishing/tests/mod.rs | 81 + .../tests/notifications.rs | 164 ++ .../document_publishing/tests/publishing.rs | 211 ++ .../workflow.rs} | 0 crates/net/src/{domain/mod.rs => domain.rs} | 12 +- crates/net/src/domain/net_event_batch.rs | 583 ---- crates/net/src/event_buffer/actor.rs | 124 + crates/net/src/event_buffer/handlers.rs | 92 + crates/net/src/event_buffer/model.rs | 313 +++ crates/net/src/event_buffer/model_tests.rs | 276 ++ crates/net/src/event_buffer/tests.rs | 126 + .../workflow.rs} | 0 crates/net/src/event_conversion/actor.rs | 168 ++ .../workflow.rs} | 0 .../{actors/mod.rs => event_subscription.rs} | 23 +- .../actor.rs} | 4 +- .../workflow.rs} | 0 crates/net/src/lib.rs | 1 + crates/net/src/{domain => }/network_status.rs | 0 crates/net/src/network_sync/actor.rs | 163 ++ .../{domain => network_sync}/correlator.rs | 0 .../net/src/network_sync/effects/admission.rs | 53 + .../src/network_sync/effects/fetch_history.rs | 254 ++ crates/net/src/network_sync/effects/mod.rs | 11 + .../net/src/network_sync/effects/readiness.rs | 81 + .../src/network_sync/effects/rebroadcast.rs | 73 + crates/net/src/network_sync/handlers.rs | 208 ++ crates/net/src/network_sync/tests.rs | 297 +++ .../net/src/{domain => network_sync}/wire.rs | 0 .../workflow.rs} | 174 +- crates/net/src/network_sync/workflow_tests.rs | 175 ++ .../src/{domain => }/peer_failure_tracker.rs | 0 .../request/src/{actors/mod.rs => actors.rs} | 4 + crates/request/src/actors/router.rs | 389 --- .../request/src/{domain/mod.rs => domain.rs} | 5 + crates/request/src/domain/routing.rs | 444 --- .../actor.rs} | 0 .../lifecycle.rs => lifecycle/workflow.rs} | 156 +- .../request/src/lifecycle/workflow_tests.rs | 158 ++ crates/request/src/routing/actor.rs | 139 + .../src/routing/effects/build_context.rs | 35 + crates/request/src/routing/effects/mod.rs | 12 + .../request/src/routing/effects/snapshot.rs | 64 + .../src/{domain => routing}/event_buffer.rs | 0 crates/request/src/routing/handlers.rs | 84 + crates/request/src/routing/tests.rs | 93 + crates/request/src/routing/workflow.rs | 139 + crates/request/src/routing/workflow_tests.rs | 311 +++ .../slashing/src/accusation_voting/actor.rs | 114 + .../src/accusation_voting/actor_tests.rs | 196 ++ .../effects/apply_actions.rs | 73 + .../accusation_voting/effects/construction.rs | 213 ++ .../src/accusation_voting/effects}/mod.rs | 9 +- .../src/accusation_voting/handlers.rs | 134 + .../slashing/src/accusation_voting/state.rs | 82 + .../transitions/incoming_accusations.rs | 283 ++ .../transitions/initiate_accusation.rs | 248 ++ .../src/accusation_voting/transitions/mod.rs | 15 + .../transitions/reverify_proofs.rs | 140 + .../accusation_voting/transitions/setup.rs | 292 ++ .../src/accusation_voting/transitions/vote.rs | 237 ++ .../src/accusation_voting/workflow.rs | 61 + .../accusation_voting/workflow_tests/mod.rs | 123 + .../workflow_tests/outcomes.rs | 87 + .../workflow_tests/voting.rs | 138 + .../slashing/src/{actors/mod.rs => actors.rs} | 7 +- .../slashing/src/actors/accusation_manager.rs | 714 ----- .../actor.rs} | 0 .../workflow.rs} | 320 +-- .../commitment_consistency/workflow_tests.rs | 316 +++ crates/slashing/src/domain.rs | 10 + .../slashing/src/domain/accusation_voting.rs | 1673 ------------ crates/slashing/src/domain/mod.rs | 13 - crates/slashing/src/lib.rs | 1 + crates/slashing/src/workflow.rs | 10 + crates/sortition/src/actors.rs | 15 + crates/sortition/src/actors/sortition.rs | 689 ----- .../src/ciphernode_selection/actor.rs | 148 + .../handlers.rs} | 144 +- .../src/{domain/mod.rs => domain.rs} | 7 + crates/sortition/src/sortition/actor.rs | 254 ++ .../finalized_committee_retention.rs | 0 .../src/sortition/handlers/committee.rs | 89 + .../src/sortition/handlers/envelope.rs | 60 + .../src/sortition/handlers/lifecycle.rs | 138 + .../sortition/src/sortition/handlers/mod.rs | 11 + .../src/sortition/handlers/registry.rs | 125 + .../src/sortition/handlers/request.rs | 51 + .../{domain => sortition}/node_registry.rs | 119 +- .../src/sortition/node_registry_tests.rs | 117 + .../selection_backend.rs} | 0 .../src/{domain => sortition}/ticket.rs | 0 .../ticket_selection.rs} | 0 crates/sync/src/actors/sync.rs | 922 ------- crates/sync/src/{domain/mod.rs => domain.rs} | 6 + crates/sync/src/lib.rs | 4 +- crates/sync/src/{actors/mod.rs => runtime.rs} | 3 + .../collect_history.rs} | 0 crates/sync/src/sync/history.rs | 43 + crates/sync/src/sync/preflight.rs | 67 + .../src/{domain => sync}/schema_version.rs | 0 crates/sync/src/sync/service.rs | 255 ++ .../snapshot_meta.rs => sync/state.rs} | 0 crates/sync/src/sync/tests/gates.rs | 222 ++ crates/sync/src/sync/tests/history.rs | 36 + crates/sync/src/sync/tests/mod.rs | 55 + crates/sync/src/sync/tests/replay.rs | 152 ++ crates/sync/src/sync/tests/schema.rs | 127 + .../sync_planner.rs => sync/workflow.rs} | 0 .../src/{actors => }/accusation_manager.rs | 0 .../{actors => }/accusation_manager_ext.rs | 0 crates/zk-prover/src/actor_system.rs | 85 + crates/zk-prover/src/actors.rs | 38 + crates/zk-prover/src/actors/mod.rs | 140 - .../src/actors/node_proof_aggregator.rs | 717 ----- crates/zk-prover/src/actors/proof_request.rs | 1444 ---------- .../src/actors/proof_verification.rs | 618 ----- .../src/actors/share_verification.rs | 810 ------ .../commitment_consistency_checker.rs | 0 .../commitment_consistency_checker_ext.rs | 0 .../{actors => }/commitment_links/c0_to_c3.rs | 0 .../{actors => }/commitment_links/c1_to_c2.rs | 0 .../{actors => }/commitment_links/c1_to_c5.rs | 0 .../{actors => }/commitment_links/c2_to_c3.rs | 0 .../c2_to_c4/mod.rs} | 172 +- .../src/commitment_links/c2_to_c4/tests.rs | 175 ++ .../commitment_links/c4a_to_c6.rs | 0 .../commitment_links/c4b_to_c6.rs | 0 .../{actors => }/commitment_links/c6_to_c7.rs | 0 .../src/{actors => }/commitment_links/mod.rs | 0 .../src/{domain/mod.rs => domain.rs} | 8 +- crates/zk-prover/src/domain/proof_request.rs | 545 ---- .../src/domain/share_verification.rs | 1213 --------- crates/zk-prover/src/lib.rs | 10 +- .../src/node_proof_aggregation/actor.rs | 299 +++ .../src/node_proof_aggregation/effects.rs | 309 +++ .../src/node_proof_aggregation/handlers.rs | 136 + .../workflow.rs} | 0 crates/zk-prover/src/proof_request/actor.rs | 106 + .../src/proof_request/actor_tests.rs | 101 + .../effects/aggregation_proofs.rs | 180 ++ .../effects/decryption_key_proofs.rs | 183 ++ .../effects/decryption_share_proofs.rs | 121 + .../src/proof_request/effects/dkg_proofs.rs | 170 ++ .../effects/encryption_key_result.rs | 118 + .../src/proof_request/effects/failures.rs | 84 + .../src/proof_request/effects/mod.rs | 18 + .../effects/publish_threshold_shares.rs | 200 ++ .../src/proof_request/effects/signing.rs | 141 + .../zk-prover/src/proof_request/handlers.rs | 141 + crates/zk-prover/src/proof_request/state.rs | 193 ++ .../src/proof_request/transitions.rs | 101 + .../zk-prover/src/proof_request/workflow.rs | 40 + .../src/proof_request/workflow_tests.rs | 237 ++ .../zk-prover/src/proof_verification/actor.rs | 159 ++ .../src/proof_verification/actor_tests.rs | 177 ++ .../src/proof_verification/effects.rs | 128 + .../src/proof_verification/handlers.rs | 175 ++ .../workflow.rs} | 0 .../zk-prover/src/share_verification/actor.rs | 122 + .../src/share_verification/actor_tests.rs | 117 + .../effects/complete_verification.rs | 26 + .../share_verification/effects/consistency.rs | 159 ++ .../share_verification/effects/dispatch.rs | 56 + .../src/share_verification/effects/ecdsa.rs | 107 + .../src/share_verification/effects/mod.rs | 15 + .../src/share_verification/effects/results.rs | 185 ++ .../src/share_verification/handlers.rs | 93 + .../zk-prover/src/share_verification/state.rs | 144 + .../transitions/consistency.rs | 23 + .../share_verification/transitions/ecdsa.rs | 188 ++ .../src/share_verification/transitions/mod.rs | 12 + .../share_verification/transitions/prepare.rs | 208 ++ .../share_verification/transitions/tally.rs | 99 + .../src/share_verification/workflow.rs | 38 + .../workflow_tests/ecdsa.rs | 144 + .../share_verification/workflow_tests/mod.rs | 95 + .../workflow_tests/shape.rs | 203 ++ .../workflow_tests/tally.rs | 140 + crates/zk-prover/src/workflow.rs | 12 + crates/zk-prover/src/{actors => }/zk_actor.rs | 0 321 files changed, 26538 insertions(+), 24932 deletions(-) create mode 100644 agent/ACTOR_AUDIT.md rename crates/aggregator/src/{actors/mod.rs => actors.rs} (57%) delete mode 100644 crates/aggregator/src/actors/publickey_aggregator.rs delete mode 100644 crates/aggregator/src/actors/threshold_plaintext_aggregator.rs rename crates/aggregator/src/{domain => }/committee.rs (100%) rename crates/aggregator/src/{actors/committee_finalizer.rs => committee_finalization/actor.rs} (53%) create mode 100644 crates/aggregator/src/committee_finalization/handlers.rs rename crates/aggregator/src/{domain => }/committee_hash.rs (100%) create mode 100644 crates/aggregator/src/domain.rs delete mode 100644 crates/aggregator/src/domain/mod.rs delete mode 100644 crates/aggregator/src/domain/publickey_aggregation.rs delete mode 100644 crates/aggregator/src/domain/threshold_plaintext_aggregation.rs rename crates/aggregator/src/{domain => }/failover.rs (100%) create mode 100644 crates/aggregator/src/plaintext_aggregation/actor.rs rename crates/aggregator/src/{actors/decryptionshare_created_buffer.rs => plaintext_aggregation/decryption_share_buffer.rs} (100%) create mode 100644 crates/aggregator/src/plaintext_aggregation/effects/mod.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/effects/prove_plaintext.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/effects/publish_result.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/handlers.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/intents.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/state.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/tests/completion.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/tests/failures.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/tests/mod.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/transitions.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/workflow.rs create mode 100644 crates/aggregator/src/plaintext_aggregation/workflow_tests.rs create mode 100644 crates/aggregator/src/public_key_aggregation/actor.rs create mode 100644 crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs create mode 100644 crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs create mode 100644 crates/aggregator/src/public_key_aggregation/effects/fold_node_proofs.rs create mode 100644 crates/aggregator/src/public_key_aggregation/effects/handle_compute_results.rs create mode 100644 crates/aggregator/src/public_key_aggregation/effects/mod.rs create mode 100644 crates/aggregator/src/public_key_aggregation/effects/publish_result.rs create mode 100644 crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs create mode 100644 crates/aggregator/src/public_key_aggregation/handlers.rs rename crates/aggregator/src/{actors/keyshare_created_filter_buffer.rs => public_key_aggregation/keyshare_buffer.rs} (100%) create mode 100644 crates/aggregator/src/public_key_aggregation/state.rs create mode 100644 crates/aggregator/src/public_key_aggregation/tests/attestations.rs create mode 100644 crates/aggregator/src/public_key_aggregation/tests/failures.rs create mode 100644 crates/aggregator/src/public_key_aggregation/tests/mod.rs create mode 100644 crates/aggregator/src/public_key_aggregation/transitions.rs create mode 100644 crates/aggregator/src/public_key_aggregation/validation.rs create mode 100644 crates/aggregator/src/public_key_aggregation/workflow.rs create mode 100644 crates/aggregator/src/public_key_aggregation/workflow_tests.rs create mode 100644 crates/aggregator/src/workflow.rs rename crates/evm/src/{actors/mod.rs => actors.rs} (68%) delete mode 100644 crates/evm/src/actors/ciphernode_registry_sol.rs delete mode 100644 crates/evm/src/actors/evm_chain_gateway.rs delete mode 100644 crates/evm/src/actors/evm_read_interface.rs delete mode 100644 crates/evm/src/actors/log_fetcher.rs create mode 100644 crates/evm/src/adapters.rs rename crates/evm/src/{actors/bonding_registry_sol.rs => bonding_registry/actor.rs} (100%) rename crates/evm/src/{domain/bonding_registry_events.rs => bonding_registry/events.rs} (100%) create mode 100644 crates/evm/src/chain_gateway/actor.rs create mode 100644 crates/evm/src/chain_gateway/handlers.rs rename crates/evm/src/{domain/chain_sync_state.rs => chain_gateway/state.rs} (100%) create mode 100644 crates/evm/src/chain_gateway/tests.rs rename crates/evm/src/{actors/evm_hub.rs => chain_hub.rs} (100%) create mode 100644 crates/evm/src/chain_reader/actor.rs rename crates/evm/src/{domain => chain_reader}/backoff.rs (100%) create mode 100644 crates/evm/src/chain_reader/effects/mod.rs create mode 100644 crates/evm/src/chain_reader/effects/provider_recovery.rs create mode 100644 crates/evm/src/chain_reader/effects/stream.rs create mode 100644 crates/evm/src/chain_reader/handlers.rs rename crates/evm/src/{domain => chain_reader}/log_timestamp.rs (100%) rename crates/evm/src/{domain => chain_reader}/reorg.rs (100%) rename crates/evm/src/{actors/sync_start_extractor.rs => chain_sync/start_extractor.rs} (100%) create mode 100644 crates/evm/src/ciphernode_registry/actor.rs create mode 100644 crates/evm/src/ciphernode_registry/effects.rs rename crates/evm/src/{domain/ciphernode_registry_events.rs => ciphernode_registry/events.rs} (100%) create mode 100644 crates/evm/src/ciphernode_registry/handlers.rs rename crates/evm/src/{domain/mod.rs => domain.rs} (62%) rename crates/evm/src/{actors/evm_parser.rs => event_decoding/actor.rs} (100%) rename crates/evm/src/{domain/evm_event_catalog.rs => event_decoding/catalog.rs} (100%) rename crates/evm/src/{domain/error_decoder.rs => event_decoding/error.rs} (100%) rename crates/evm/src/{domain/evm_log_observation.rs => event_decoding/observation.rs} (100%) rename crates/evm/src/{actors/evm_router.rs => event_router.rs} (100%) rename crates/evm/src/{actors/fix_historical_order.rs => historical_order/actor.rs} (100%) rename crates/evm/src/{domain/historical_order_fixer.rs => historical_order/workflow.rs} (100%) rename crates/evm/src/{domain/interfold_events.rs => interfold/events.rs} (100%) rename crates/evm/src/{actors/interfold_sol_reader.rs => interfold/reader.rs} (100%) create mode 100644 crates/evm/src/interfold_writing/actor.rs create mode 100644 crates/evm/src/interfold_writing/effects.rs rename crates/evm/src/{actors/interfold_sol_writer.rs => interfold_writing/handlers.rs} (55%) rename crates/evm/src/{domain/plaintext_publication.rs => interfold_writing/workflow.rs} (100%) create mode 100644 crates/evm/src/log_fetching/adapter.rs create mode 100644 crates/evm/src/log_fetching/tests/backfill.rs create mode 100644 crates/evm/src/log_fetching/tests/fetch.rs create mode 100644 crates/evm/src/log_fetching/tests/mod.rs rename crates/evm/src/{domain/slashing_events.rs => slashing/events.rs} (100%) rename crates/evm/src/{domain/attestation_evidence.rs => slashing/evidence.rs} (100%) rename crates/evm/src/{actors/slashing_manager_sol_reader.rs => slashing/reader.rs} (100%) create mode 100644 crates/evm/src/slashing_writing/actor.rs create mode 100644 crates/evm/src/slashing_writing/effects.rs rename crates/evm/src/{actors/slashing_manager_sol_writer.rs => slashing_writing/handlers.rs} (51%) rename crates/evm/src/{domain/slash_submission.rs => slashing_writing/workflow.rs} (100%) rename crates/keyshare/src/{actors/mod.rs => actors.rs} (65%) delete mode 100644 crates/keyshare/src/actors/threshold_keyshare.rs rename crates/keyshare/src/{domain/mod.rs => domain.rs} (62%) create mode 100644 crates/keyshare/src/threshold_keyshare/actor.rs rename crates/keyshare/src/{domain/decryption_key_shared_collection.rs => threshold_keyshare/collect_decryption_keys.rs} (100%) rename crates/keyshare/src/{domain/encryption_key_collection.rs => threshold_keyshare/collect_encryption_keys.rs} (100%) rename crates/keyshare/src/{domain/threshold_share_collection.rs => threshold_keyshare/collect_threshold_shares.rs} (100%) rename crates/keyshare/src/{actors/decryption_key_shared_collector.rs => threshold_keyshare/decryption_key_collector.rs} (100%) rename crates/keyshare/src/{domain/decryption_key_calculation.rs => threshold_keyshare/derive_decryption_key.rs} (100%) create mode 100644 crates/keyshare/src/threshold_keyshare/effects/calculate_decryption_key.rs create mode 100644 crates/keyshare/src/threshold_keyshare/effects/coordinate_collectors.rs create mode 100644 crates/keyshare/src/threshold_keyshare/effects/create_decryption_share.rs create mode 100644 crates/keyshare/src/threshold_keyshare/effects/generate_threshold_share.rs create mode 100644 crates/keyshare/src/threshold_keyshare/effects/initialize_dkg.rs create mode 100644 crates/keyshare/src/threshold_keyshare/effects/mod.rs create mode 100644 crates/keyshare/src/threshold_keyshare/effects/route_events.rs create mode 100644 crates/keyshare/src/threshold_keyshare/effects/track_proofs.rs create mode 100644 crates/keyshare/src/threshold_keyshare/effects/verify_decryption_key.rs create mode 100644 crates/keyshare/src/threshold_keyshare/effects/verify_threshold_shares.rs rename crates/keyshare/src/{actors => threshold_keyshare}/encryption_key_collector.rs (100%) rename crates/keyshare/src/{domain/bfv_keygen.rs => threshold_keyshare/generate_encryption_key.rs} (100%) rename crates/keyshare/src/{domain/share_generation.rs => threshold_keyshare/generate_shares.rs} (100%) create mode 100644 crates/keyshare/src/threshold_keyshare/handlers.rs rename crates/keyshare/src/{domain/keyshare_state.rs => threshold_keyshare/state.rs} (67%) create mode 100644 crates/keyshare/src/threshold_keyshare/state_tests.rs create mode 100644 crates/keyshare/src/threshold_keyshare/tests.rs rename crates/keyshare/src/{actors => threshold_keyshare}/threshold_share_collector.rs (100%) rename crates/keyshare/src/{domain => threshold_keyshare}/timeout_policy.rs (100%) create mode 100644 crates/net/src/actors.rs delete mode 100644 crates/net/src/actors/document_publisher.rs delete mode 100644 crates/net/src/actors/net_event_buffer.rs delete mode 100644 crates/net/src/actors/net_sync_manager.rs create mode 100644 crates/net/src/document_publishing/actor.rs create mode 100644 crates/net/src/document_publishing/effects.rs create mode 100644 crates/net/src/document_publishing/handlers.rs create mode 100644 crates/net/src/document_publishing/tests/mod.rs create mode 100644 crates/net/src/document_publishing/tests/notifications.rs create mode 100644 crates/net/src/document_publishing/tests/publishing.rs rename crates/net/src/{domain/document_publishing.rs => document_publishing/workflow.rs} (100%) rename crates/net/src/{domain/mod.rs => domain.rs} (69%) delete mode 100644 crates/net/src/domain/net_event_batch.rs create mode 100644 crates/net/src/event_buffer/actor.rs create mode 100644 crates/net/src/event_buffer/handlers.rs create mode 100644 crates/net/src/event_buffer/model.rs create mode 100644 crates/net/src/event_buffer/model_tests.rs create mode 100644 crates/net/src/event_buffer/tests.rs rename crates/net/src/{domain/net_buffer.rs => event_buffer/workflow.rs} (100%) create mode 100644 crates/net/src/event_conversion/actor.rs rename crates/net/src/{domain/event_conversion.rs => event_conversion/workflow.rs} (100%) rename crates/net/src/{actors/mod.rs => event_subscription.rs} (80%) rename crates/net/src/{actors/net_event_translator.rs => event_translation/actor.rs} (96%) rename crates/net/src/{domain/event_translation.rs => event_translation/workflow.rs} (100%) rename crates/net/src/{domain => }/network_status.rs (100%) create mode 100644 crates/net/src/network_sync/actor.rs rename crates/net/src/{domain => network_sync}/correlator.rs (100%) create mode 100644 crates/net/src/network_sync/effects/admission.rs create mode 100644 crates/net/src/network_sync/effects/fetch_history.rs create mode 100644 crates/net/src/network_sync/effects/mod.rs create mode 100644 crates/net/src/network_sync/effects/readiness.rs create mode 100644 crates/net/src/network_sync/effects/rebroadcast.rs create mode 100644 crates/net/src/network_sync/handlers.rs create mode 100644 crates/net/src/network_sync/tests.rs rename crates/net/src/{domain => network_sync}/wire.rs (100%) rename crates/net/src/{domain/sync_coordinator.rs => network_sync/workflow.rs} (50%) create mode 100644 crates/net/src/network_sync/workflow_tests.rs rename crates/net/src/{domain => }/peer_failure_tracker.rs (100%) rename crates/request/src/{actors/mod.rs => actors.rs} (65%) delete mode 100644 crates/request/src/actors/router.rs rename crates/request/src/{domain/mod.rs => domain.rs} (63%) delete mode 100644 crates/request/src/domain/routing.rs rename crates/request/src/{actors/lifecycle_coordinator.rs => lifecycle/actor.rs} (100%) rename crates/request/src/{domain/lifecycle.rs => lifecycle/workflow.rs} (57%) create mode 100644 crates/request/src/lifecycle/workflow_tests.rs create mode 100644 crates/request/src/routing/actor.rs create mode 100644 crates/request/src/routing/effects/build_context.rs create mode 100644 crates/request/src/routing/effects/mod.rs create mode 100644 crates/request/src/routing/effects/snapshot.rs rename crates/request/src/{domain => routing}/event_buffer.rs (100%) create mode 100644 crates/request/src/routing/handlers.rs create mode 100644 crates/request/src/routing/tests.rs create mode 100644 crates/request/src/routing/workflow.rs create mode 100644 crates/request/src/routing/workflow_tests.rs create mode 100644 crates/slashing/src/accusation_voting/actor.rs create mode 100644 crates/slashing/src/accusation_voting/actor_tests.rs create mode 100644 crates/slashing/src/accusation_voting/effects/apply_actions.rs create mode 100644 crates/slashing/src/accusation_voting/effects/construction.rs rename crates/{sortition/src/actors => slashing/src/accusation_voting/effects}/mod.rs (66%) create mode 100644 crates/slashing/src/accusation_voting/handlers.rs create mode 100644 crates/slashing/src/accusation_voting/state.rs create mode 100644 crates/slashing/src/accusation_voting/transitions/incoming_accusations.rs create mode 100644 crates/slashing/src/accusation_voting/transitions/initiate_accusation.rs create mode 100644 crates/slashing/src/accusation_voting/transitions/mod.rs create mode 100644 crates/slashing/src/accusation_voting/transitions/reverify_proofs.rs create mode 100644 crates/slashing/src/accusation_voting/transitions/setup.rs create mode 100644 crates/slashing/src/accusation_voting/transitions/vote.rs create mode 100644 crates/slashing/src/accusation_voting/workflow.rs create mode 100644 crates/slashing/src/accusation_voting/workflow_tests/mod.rs create mode 100644 crates/slashing/src/accusation_voting/workflow_tests/outcomes.rs create mode 100644 crates/slashing/src/accusation_voting/workflow_tests/voting.rs rename crates/slashing/src/{actors/mod.rs => actors.rs} (57%) delete mode 100644 crates/slashing/src/actors/accusation_manager.rs rename crates/slashing/src/{actors/commitment_consistency_checker.rs => commitment_consistency/actor.rs} (100%) rename crates/slashing/src/{domain/commitment_consistency.rs => commitment_consistency/workflow.rs} (64%) create mode 100644 crates/slashing/src/commitment_consistency/workflow_tests.rs create mode 100644 crates/slashing/src/domain.rs delete mode 100644 crates/slashing/src/domain/accusation_voting.rs delete mode 100644 crates/slashing/src/domain/mod.rs create mode 100644 crates/slashing/src/workflow.rs create mode 100644 crates/sortition/src/actors.rs delete mode 100644 crates/sortition/src/actors/sortition.rs create mode 100644 crates/sortition/src/ciphernode_selection/actor.rs rename crates/sortition/src/{actors/ciphernode_selector.rs => ciphernode_selection/handlers.rs} (64%) rename crates/sortition/src/{domain/mod.rs => domain.rs} (61%) create mode 100644 crates/sortition/src/sortition/actor.rs rename crates/sortition/src/{domain => sortition}/finalized_committee_retention.rs (100%) create mode 100644 crates/sortition/src/sortition/handlers/committee.rs create mode 100644 crates/sortition/src/sortition/handlers/envelope.rs create mode 100644 crates/sortition/src/sortition/handlers/lifecycle.rs create mode 100644 crates/sortition/src/sortition/handlers/mod.rs create mode 100644 crates/sortition/src/sortition/handlers/registry.rs create mode 100644 crates/sortition/src/sortition/handlers/request.rs rename crates/sortition/src/{domain => sortition}/node_registry.rs (69%) create mode 100644 crates/sortition/src/sortition/node_registry_tests.rs rename crates/sortition/src/{domain/backends.rs => sortition/selection_backend.rs} (100%) rename crates/sortition/src/{domain => sortition}/ticket.rs (100%) rename crates/sortition/src/{domain/ticket_sortition.rs => sortition/ticket_selection.rs} (100%) delete mode 100644 crates/sync/src/actors/sync.rs rename crates/sync/src/{domain/mod.rs => domain.rs} (73%) rename crates/sync/src/{actors/mod.rs => runtime.rs} (72%) rename crates/sync/src/{domain/historical_evm_collector.rs => sync/collect_history.rs} (100%) create mode 100644 crates/sync/src/sync/history.rs create mode 100644 crates/sync/src/sync/preflight.rs rename crates/sync/src/{domain => sync}/schema_version.rs (100%) create mode 100644 crates/sync/src/sync/service.rs rename crates/sync/src/{domain/snapshot_meta.rs => sync/state.rs} (100%) create mode 100644 crates/sync/src/sync/tests/gates.rs create mode 100644 crates/sync/src/sync/tests/history.rs create mode 100644 crates/sync/src/sync/tests/mod.rs create mode 100644 crates/sync/src/sync/tests/replay.rs create mode 100644 crates/sync/src/sync/tests/schema.rs rename crates/sync/src/{domain/sync_planner.rs => sync/workflow.rs} (100%) rename crates/zk-prover/src/{actors => }/accusation_manager.rs (100%) rename crates/zk-prover/src/{actors => }/accusation_manager_ext.rs (100%) create mode 100644 crates/zk-prover/src/actor_system.rs create mode 100644 crates/zk-prover/src/actors.rs delete mode 100644 crates/zk-prover/src/actors/mod.rs delete mode 100644 crates/zk-prover/src/actors/node_proof_aggregator.rs delete mode 100644 crates/zk-prover/src/actors/proof_request.rs delete mode 100644 crates/zk-prover/src/actors/proof_verification.rs delete mode 100644 crates/zk-prover/src/actors/share_verification.rs rename crates/zk-prover/src/{actors => }/commitment_consistency_checker.rs (100%) rename crates/zk-prover/src/{actors => }/commitment_consistency_checker_ext.rs (100%) rename crates/zk-prover/src/{actors => }/commitment_links/c0_to_c3.rs (100%) rename crates/zk-prover/src/{actors => }/commitment_links/c1_to_c2.rs (100%) rename crates/zk-prover/src/{actors => }/commitment_links/c1_to_c5.rs (100%) rename crates/zk-prover/src/{actors => }/commitment_links/c2_to_c3.rs (100%) rename crates/zk-prover/src/{actors/commitment_links/c2_to_c4.rs => commitment_links/c2_to_c4/mod.rs} (55%) create mode 100644 crates/zk-prover/src/commitment_links/c2_to_c4/tests.rs rename crates/zk-prover/src/{actors => }/commitment_links/c4a_to_c6.rs (100%) rename crates/zk-prover/src/{actors => }/commitment_links/c4b_to_c6.rs (100%) rename crates/zk-prover/src/{actors => }/commitment_links/c6_to_c7.rs (100%) rename crates/zk-prover/src/{actors => }/commitment_links/mod.rs (100%) rename crates/zk-prover/src/{domain/mod.rs => domain.rs} (68%) delete mode 100644 crates/zk-prover/src/domain/proof_request.rs delete mode 100644 crates/zk-prover/src/domain/share_verification.rs create mode 100644 crates/zk-prover/src/node_proof_aggregation/actor.rs create mode 100644 crates/zk-prover/src/node_proof_aggregation/effects.rs create mode 100644 crates/zk-prover/src/node_proof_aggregation/handlers.rs rename crates/zk-prover/src/{domain/node_dkg_fold.rs => node_proof_aggregation/workflow.rs} (100%) create mode 100644 crates/zk-prover/src/proof_request/actor.rs create mode 100644 crates/zk-prover/src/proof_request/actor_tests.rs create mode 100644 crates/zk-prover/src/proof_request/effects/aggregation_proofs.rs create mode 100644 crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs create mode 100644 crates/zk-prover/src/proof_request/effects/decryption_share_proofs.rs create mode 100644 crates/zk-prover/src/proof_request/effects/dkg_proofs.rs create mode 100644 crates/zk-prover/src/proof_request/effects/encryption_key_result.rs create mode 100644 crates/zk-prover/src/proof_request/effects/failures.rs create mode 100644 crates/zk-prover/src/proof_request/effects/mod.rs create mode 100644 crates/zk-prover/src/proof_request/effects/publish_threshold_shares.rs create mode 100644 crates/zk-prover/src/proof_request/effects/signing.rs create mode 100644 crates/zk-prover/src/proof_request/handlers.rs create mode 100644 crates/zk-prover/src/proof_request/state.rs create mode 100644 crates/zk-prover/src/proof_request/transitions.rs create mode 100644 crates/zk-prover/src/proof_request/workflow.rs create mode 100644 crates/zk-prover/src/proof_request/workflow_tests.rs create mode 100644 crates/zk-prover/src/proof_verification/actor.rs create mode 100644 crates/zk-prover/src/proof_verification/actor_tests.rs create mode 100644 crates/zk-prover/src/proof_verification/effects.rs create mode 100644 crates/zk-prover/src/proof_verification/handlers.rs rename crates/zk-prover/src/{domain/proof_verification.rs => proof_verification/workflow.rs} (100%) create mode 100644 crates/zk-prover/src/share_verification/actor.rs create mode 100644 crates/zk-prover/src/share_verification/actor_tests.rs create mode 100644 crates/zk-prover/src/share_verification/effects/complete_verification.rs create mode 100644 crates/zk-prover/src/share_verification/effects/consistency.rs create mode 100644 crates/zk-prover/src/share_verification/effects/dispatch.rs create mode 100644 crates/zk-prover/src/share_verification/effects/ecdsa.rs create mode 100644 crates/zk-prover/src/share_verification/effects/mod.rs create mode 100644 crates/zk-prover/src/share_verification/effects/results.rs create mode 100644 crates/zk-prover/src/share_verification/handlers.rs create mode 100644 crates/zk-prover/src/share_verification/state.rs create mode 100644 crates/zk-prover/src/share_verification/transitions/consistency.rs create mode 100644 crates/zk-prover/src/share_verification/transitions/ecdsa.rs create mode 100644 crates/zk-prover/src/share_verification/transitions/mod.rs create mode 100644 crates/zk-prover/src/share_verification/transitions/prepare.rs create mode 100644 crates/zk-prover/src/share_verification/transitions/tally.rs create mode 100644 crates/zk-prover/src/share_verification/workflow.rs create mode 100644 crates/zk-prover/src/share_verification/workflow_tests/ecdsa.rs create mode 100644 crates/zk-prover/src/share_verification/workflow_tests/mod.rs create mode 100644 crates/zk-prover/src/share_verification/workflow_tests/shape.rs create mode 100644 crates/zk-prover/src/share_verification/workflow_tests/tally.rs create mode 100644 crates/zk-prover/src/workflow.rs rename crates/zk-prover/src/{actors => }/zk_actor.rs (100%) diff --git a/agent/ACTOR_AUDIT.md b/agent/ACTOR_AUDIT.md new file mode 100644 index 0000000000..3896292b42 --- /dev/null +++ b/agent/ACTOR_AUDIT.md @@ -0,0 +1,65 @@ +# Ciphernode Actor Audit + +This audit records the repository-wide thin-actor and source-layout review. Read it with +[`ARCHITECTURE.md`](ARCHITECTURE.md), which defines the target, and +[`CRATES_ARCHITECTURE.md`](CRATES_ARCHITECTURE.md), which describes the implementation. + +## Method + +An actor is thin when it has one runtime reason to change: mailbox serialization, routing, +scheduling, lifecycle, or supervision. Line count is evidence, not the definition. Roughly 300 +production lines triggers a responsibility review; tests, generated bindings, and cohesive +cryptographic algorithms are judged separately. + +The audit checked all nine crates that originally contained `src/actors/`. Before this refactor, +17 production actor modules were over the review threshold; the largest were `ThresholdKeyshare` +(2,198 lines), `PublicKeyAggregator` (1,591), `ProofRequestActor` (1,344), and +`ThresholdPlaintextAggregator` (1,029). + +The first extraction made the actors thin but retained layer-first `actors/`, `workflow/`, and +`domain/` trees. That made placement hard to infer because one protocol capability was scattered +across several top-level folders. The follow-up layout groups implementation by capability and uses +the same role vocabulary inside each capability. Root compatibility files preserve established +Rust paths; they do not own implementation. Inline test suites were also moved beside the roles +they exercise so production responsibility size remains visible. + +## Findings by crate + +| Crate | Actors reviewed | Result | +| --- | --- | --- | +| `e3-aggregator` | committee finalizer, keyshare/decryption buffers, public-key aggregator, plaintext aggregator | `committee_finalization`, `public_key_aggregation`, and `plaintext_aggregation` each contain their actor, handlers, workflow/state, semantic effects, and tests. | +| `e3-evm` | parser/router/hub, chain gateway, readers, registry/interfold/slashing writers, log fetcher | Each chain capability owns its actor plus events/workflow/effects. `log_fetching` is explicitly an adapter capability; provider and transaction work is no longer hidden below actor `runtime/`. | +| `e3-keyshare` | encryption/share/decryption collectors, `ThresholdKeyshare` | The `threshold_keyshare` capability contains the request-local coordinator, collectors, persisted state, pure DKG calculations, handlers, and semantically named effect operations. Transient async-gap data remains grouped in `PendingKeyshareWork`. | +| `e3-net` | event buffer/translator, sync manager, document publisher/converter | Admission, readiness, rebroadcast, historical sync, conversion, and DHT/gossip effects are isolated. The actors now own transport ordering and lifecycle rather than document-validation policy. | +| `e3-request` | lifecycle coordinator, E3 router | `routing` and `lifecycle` contain predictable actor/workflow roles; context construction, snapshots, and the request event buffer are named beside routing instead of hidden in a generic runtime package. | +| `e3-slashing` | accusation manager, commitment consistency checker | `accusation_voting` and `commitment_consistency` each co-locate the actor shell with deterministic workflow decisions and adjacent tests. | +| `e3-sortition` | sortition, ciphernode selector | `sortition` contains actor, registry, selection backend, ticket rules, and retention; `ciphernode_selection` contains its actor and handlers. | +| `e3-sync` | bootstrap/replay functions and messages | This crate has no Actix actor. The `sync` capability honestly names its service, state, workflow, preflight, history collection, and tests. | +| `e3-zk-prover` | proof requester, share verifier, C0 verifier, node proof aggregator, ZK worker, commitment links | Proof capabilities co-locate actor, handlers, state/workflow, effects, and tests. Effect files use semantic proof-operation names instead of unexplained `c0`–`c7` filenames; pure commitment links have their own capability. | + +## State classification applied + +- Persisted workflow state: aggregation and keyshare state enums stored through repositories. +- Derivable state: canonical committee/preset caches rebuilt from repositories and replay. +- Ephemeral effect state: correlation IDs, collector addresses, timer handles, early-arrival buffers, + and in-flight submission guards. These are grouped and named rather than mixed into protocol + state. +- External authority: EVM contract state. Writer preflights provide cross-restart idempotency where + no durable local outbox exists. + +`Persistable::try_mutate` now accepts the snapshot write into the bounded store mailbox before it +exposes the new value in memory. This does not turn snapshots into an external-effect outbox; the +append-only event log and chain remain the stated authorities. Synchronous `BusHandle` publication +still uses the existing burst-tolerant `do_send` path and is recorded as backpressure debt. + +## Deliberate residuals + +This refactor does not split files only to satisfy a number. Generated ABI bindings, circuit witness +construction, and cohesive FHE/math algorithms may exceed 300 lines. Large non-actor composition or +infrastructure coordinators—notably `CiphernodeBuilder`, `NetInterface`, and the multithread task +pool—need their own behavior-preserving projects if their responsibilities are changed. + +The remaining architectural gap is durable effect intent. Transaction submission and some +cryptographic work still rely on replay plus external preflight instead of a versioned local +intent/result outbox. That is a schema and recovery change, not a safe file-movement refactor, and +must be implemented with migration and crash-matrix tests. diff --git a/agent/ARCHITECTURE.md b/agent/ARCHITECTURE.md index 0cf446f08a..e00e9a13f9 100644 --- a/agent/ARCHITECTURE.md +++ b/agent/ARCHITECTURE.md @@ -1,382 +1,505 @@ -# Interfold Ciphernode — Architecture & Contribution Guide +# Interfold Ciphernode Target Architecture -> Read this **before** writing or modifying any Rust code in `crates/`. It defines the canonical -> structure, the actor/service pattern, and the rules that keep the architecture from drifting. Also -> read `agent/RULES.md` and the relevant `agent/flow-trace/*.md` for protocol behavior. +This document defines the target architecture for the Rust ciphernode. It is a design constraint for +new code and the destination for incremental refactors; it is not a claim that every existing module +already complies. ---- +For the current implementation and protocol sequence, read +[`CRATES_ARCHITECTURE.md`](CRATES_ARCHITECTURE.md) and [`flow-trace/`](flow-trace/). When documents +disagree with deployed contracts or executable protocol tests, the contracts and tests win and the +documents must be corrected. -## 1. The Core Principle: Actors are Transport, Services are Logic +The repository-wide thin-actor findings and deliberate residuals are recorded in +[`ACTOR_AUDIT.md`](ACTOR_AUDIT.md). -The ciphernode is an [actix](https://docs.rs/actix) actor system wired around a central pub/sub -`EventBus`. The single most important architectural rule: +## Why Interfold Uses Actors -> **Actors do message passing ONLY. ALL business logic lives in plain, sync, actor-free service -> structs.** +Interfold is an asynchronous, multi-party cryptographic workflow. A node concurrently receives +chain logs, peer messages, timers, proof results, storage acknowledgements, and operator signals. +Work is partitioned by E3 and must remain responsive while cryptographic jobs run for seconds or +minutes. Actors are a good fit for those concurrency boundaries. -An actor's `Handler` is a thin shell that: +Actors are not the domain model. Putting every rule, state transition, and external effect in an +Actix handler creates large stateful objects that are hard to test and impossible to replay with +confidence. -1. Destructures the incoming event/message. -2. Calls a pure service method. -3. Applies the returned decision (persist state, publish events, schedule timers). +The governing principle is: -It contains **no** validation, no crypto, no aggregation, no state-transition rules, no math. Those -belong in the service. +> Actors are concurrency boundaries. Deterministic reducers own protocol and workflow decisions. +> Effect runners perform crypto, storage, network, and chain I/O. Every correctness-relevant +> transition and intent is recoverable. -### Why +This means Interfold keeps global choreography between nodes while using explicit, persisted local +orchestration for each E3. -- **Testability** — services are unit-tested synchronously with no actix runtime, no bus, no store. - Fast, deterministic, exhaustive. -- **Reasoning** — protocol logic reads top-to-bottom in one place instead of being scattered across - `Handler` impls and implicit event dispatch. -- **Resilience** — pure services with explicit decision enums make replay, restart, and - snapshot/hydrate behavior predictable. +## Sources of Authority -### The shape of a thin actor handler +In descending order: -```rust -impl Handler for ThresholdKeyshare { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) { - let (data, ec) = msg.into_components(); - match data { - InterfoldEventData::E3Requested(d) => { - // 1. call pure service -> decision - let outcome = self.state_service.on_e3_requested(d); - // 2. apply decision (I/O only) - self.apply(outcome, &ec, ctx); - } - _ => (), - } - } -} -``` +1. Deployed contract behavior and protocol/circuit invariants. +2. Compatibility and end-to-end tests. +3. Durable event and snapshot schemas already used by production nodes. +4. [`flow-trace/`](flow-trace/) and [`CRATES_ARCHITECTURE.md`](CRATES_ARCHITECTURE.md). +5. This target document. -### The shape of a pure service +A cleanup must never silently change committee ordering, threshold meaning, proof multiplicity, +hashing, signatures, circuit witness shape, event identity, or replay semantics. Such a change is a +protocol migration and requires explicit versioning and compatibility tests. -```rust -// NO actix, NO Persistable, NO BusHandle, NO Addr in this file. -// `tracing` logging is allowed. -pub struct EncryptionKeyCollection { /* plain fields */ } +## Canonical Module Structure (Normative) -pub enum CollectOutcome { Ignored, Pending, Completed(Bundle) } +The filesystem is organized by capability first and role second: -impl EncryptionKeyCollection { - pub fn collect(&mut self, share: EncryptionKeyShare) -> CollectOutcome { ... } -} +> Folders answer “what protocol capability am I changing?” Files answer “what role does this code +> play in that capability?” + +This keeps all code needed to understand one protocol slice together. It also avoids top-level +layer directories whose contents grow into unrelated lists of actor names, workflow names, and +domain names. -#[cfg(test)] -mod tests { /* sync unit tests against the service */ } +```text +crates//src/ + lib.rs + .rs + / + actor.rs + handlers.rs + state.rs + workflow.rs + transitions.rs + intents.rs + effects.rs + validation.rs + tests.rs + messages.rs + repo.rs ``` ---- +Only create the files a capability needs. A pure capability that fits in one file stays at the +crate root, for example `network_status.rs`. Once it needs multiple roles, create one capability +directory and move the implementation into it. Do not create an empty directory hierarchy in +advance. + +The standard role names are: + +| File | Owns | +| --- | --- | +| `actor.rs` | actor identity, construction, owned runtime state, lifecycle, and public surface | +| `handlers.rs` | mailbox entry points and outer-envelope validation | +| `state.rs` | durable or derivable capability state and accepted input values | +| `workflow.rs` | deterministic decisions, transitions, and intent production | +| `transitions.rs` | reducer implementation when separating it makes `workflow.rs` a clearer surface | +| `intents.rs` | typed effect intents when they would obscure the workflow | +| `effects.rs` | persistence, crypto workers, bus publication, timers, network, chain, and process I/O | +| `validation.rs` | reusable pure protocol validation | +| `tests.rs` | focused capability tests when only one suite is needed | + +Use `handlers/`, `effects/`, `transitions/`, or `tests/` only after the corresponding single file has several +independent concerns. Files below those directories use semantic operation names such as +`publish_result.rs`, `verify_key_proofs.rs`, or `fetch_history.rs`. Stage labels such as `c1.rs`, +`c5.rs`, and generic names such as `runtime.rs`, `helpers.rs`, `logic.rs`, or `utils.rs` are not +acceptable substitutes for a responsibility. + +When a capability needs separate suites for separate roles, name them after the role: +`actor_tests.rs`, `workflow_tests.rs`, `state_tests.rs`, or the corresponding directory form. This +is preferable to several unrelated test modules hidden behind one generic `tests.rs`. + +A specialized pure algorithm may use a semantic filename beside the standard roles when forcing it +into `workflow.rs` or `validation.rs` would hide what it does. Examples include +`derive_decryption_key.rs` and `ticket_selection.rs`. Its name must describe the protocol operation, +not its implementation mechanism or historical location. + +### Where a new file goes + +Use this decision order: + +1. Identify the protocol capability that owns the behavior. Extend that directory; do not choose a + top-level layer first. +2. If the behavior decides what should happen from explicit inputs, put it in `workflow.rs`, + `validation.rs`, or a named pure algorithm file. +3. If it performs I/O or dispatches work, put it in `effects.rs` (or a semantic file below + `effects/`). +4. If it receives an Actix message, put the entry point in `handlers.rs`; actor construction and + lifecycle stay in `actor.rs`. +5. If no existing capability owns it, start with `.rs`. Promote it to a directory only + when a second role appears. + +`lib.rs` declares modules, composes the public surface, and re-exports stable APIs. Crate-wide +`messages.rs` and `repo.rs` are allowed when several capabilities genuinely share that vocabulary. +Concrete cross-capability adapters or startup composition may also remain at the crate root, but a +one-capability adapter belongs with its capability. + +Root files named `actors.rs`, `domain.rs`, `workflow.rs`, `adapters.rs`, or `runtime.rs` may exist as +temporary compatibility views. They use `#[path = "..."]` declarations to preserve established +Rust module paths while implementation files live in capability directories. They must not acquire +new business logic and must not grow back into layer directories. + +### Responsibility and dependency rules + +Business logic lives with its capability. Pure protocol rules and calculations do not know which +actor invoked them; workflows deterministically turn state plus input into a new state and typed +intents. Actors apply those decisions and effects execute the resulting I/O. + +```text +/actor.rs + handlers.rs ──► workflow.rs ──► state.rs / validation.rs + │ │ + └──────────────────────────► effects.rs ──► external systems +``` + +- `state`, `validation`, workflows, and pure algorithm files must not depend on Actix, + repositories, network clients, wall-clock calls, or process execution. +- A workflow must not call concrete I/O. It returns explicit intents or typed decisions. +- Actors and handlers own serialization, scheduling, supervision, and dispatch; they do not own + threshold rules, canonical ordering, proof validity, or state-machine legality. +- Effects translate typed intents into bounded concrete work. They do not decide protocol + progression. +- Compatibility views may expose old module paths but may not invert these dependencies. + +Cross-crate dependencies must follow the workspace layering documented in +[`CRATES_ARCHITECTURE.md`](CRATES_ARCHITECTURE.md). A lower-level crate must not import an +actor-bearing orchestration crate merely to reuse a value type. + +## Domain Layer + +The domain layer owns rules whose result depends only on its inputs, including: + +- threshold and committee calculations; +- canonical party, score, proof, and ciphertext ordering; +- membership and multiplicity validation; +- commitment/link consistency rules; +- state-machine legality; +- stable IDs, hashes, and signature preimages; +- typed decisions such as `Accept`, `IgnoreDuplicate`, `Reject`, `FailE3`, or `Accuse`. + +Domain functions return typed errors and decisions. They do not publish events, log as their only +error handling, read the clock, or mutate actor state. + +Protocol-specific invariants must be named and tested. Important examples include: + +- runtime `party_id` is derived from the finalized committee normalized by ascending ticket score; +- the active aggregator is the lowest non-expelled `party_id`; +- the DKG aggregation circuit receives exactly `H` canonical honest NodeFold proofs and exactly `N` + ordered committee addresses; +- C2a/C2b are singleton proofs, while C3a/C3b follow the configured recipient/row multiplicities; +- TrBFV and Noir witness dimensions come from the active preset, never from incidental vector size. + +## Workflow Layer + +Long-running protocol behavior is modeled as a deterministic transition: -## 2. Canonical Crate Layout +```rust,ignore +pub struct Transition { + pub state: S, + pub intents: Vec, +} -Every **actor-bearing** crate uses this layout. No exceptions without a written justification in the -crate's module docs. +pub trait Workflow { + type State; + type Input; + type Intent; + type Error; + fn reduce( + state: &Self::State, + input: Self::Input, + ) -> Result, Self::Error>; +} ``` -crates//src/ - lib.rs # module decls + `pub use` re-exports ONLY. Public API surface. - actors/ - mod.rs # `mod x; pub use x::*;` - .rs # thin actix Actor + Handler shells (NO business logic) - domain/ - mod.rs # `mod x; pub use x::*;` - .rs # pure sync service + its state types + its #[cfg(test)] tests - messages.rs # actor Message types / rtypes (or messages/ dir if many) - repo.rs # Repository factory traits for this crate's persisted state - ext.rs # E3Extension impl + hydrate path, if the crate plugs into E3Router - .rs # config.rs, backends.rs, etc. as needed + +An input is a fact already accepted at a trust boundary: a chain observation, verified peer message, +timer firing, effect result, or operator command. An intent describes work to perform, for example: + +```rust,ignore +enum DkgIntent { + PersistDeadline { deadline: Timestamp }, + GenerateEncryptionKey { operation_id: OperationId }, + VerifyShareBundle { operation_id: OperationId, party_id: PartyId }, + BroadcastThresholdShare { operation_id: OperationId }, + PublishFailure { operation_id: OperationId, reason: FailureReason }, +} ``` -**Leaf / library crates (no actors)** do not need the `actors/`+`domain/` split, but still follow: -**one major struct → one file**; `lib.rs` only wires and re-exports; no dead/old files. +Reducers may update persisted workflow state and emit zero or more intents. They do not await, +spawn, address actors, or execute an intent themselves. + +Each intent that can change protocol outcome has: + +- a stable operation/idempotency key derived from E3, stage, party, artifact type, and index; +- enough versioned data to retry after restart; +- an explicit result type; +- a retry classification (`never`, `bounded`, `until deadline`, or `operator intervention`); +- a terminal failure mapping where retry cannot restore progress. + +## Actor Layer + +An actor owns serialized access to one runtime partition. It may: + +- receive and authenticate messages; +- preserve per-E3 ordering; +- load and persist workflow state; +- call a reducer; +- durably record transitions and intents; +- dispatch committed intents to effect runners; +- apply correlated results; +- schedule persisted deadlines; +- cancel work and stop child actors; +- expose health, queue, and progress signals; +- supervise or recreate owned children. + +An actor handler must not: + +- perform BFV/TrBFV/Noir calculations; +- execute `bb`, EVM RPC, libp2p, filesystem, or repository operations inline; +- encode protocol validity as an unstructured sequence of mutations and `do_send` calls; +- ignore a full or closed correctness-critical mailbox; +- keep restart-critical progress solely in memory; +- use detached tasks with no owner, cancellation, or shutdown barrier. + +“Thin” is about responsibility, not line count. A handler that validates an envelope, invokes one +transition, commits it, and dispatches its intents is thin even if the surrounding actor contains +careful runtime plumbing. A 20-line handler that performs an irreversible unacknowledged send is not +architecturally sound. + +## Adapter and Effect Layer + +Effect runners own concrete side effects: + +- BFV/TrBFV computation; +- ZK proving and verification processes; +- EventStore and snapshot storage; +- EVM reads and transaction submission; +- libp2p publication and synchronization; +- clocks and timers; +- local secret encryption and filesystem access. + +Heavy computation runs in bounded worker pools, never in an actor mailbox thread. Pools are bounded +by both job count and estimated bytes. Jobs report correlation ID, operation ID, success/failure, +timing, and cancellation outcome. + +Adapters translate external representations into domain values and enforce the boundary's trust +policy. Peer identity, committee membership, claimed party slot, signature, chain ID, E3 ID, proof +type, payload size, and schema version are checked before a message can drive a workflow. + +## Durable Processing Model + +The target event path is: + +```text +EVM / libp2p / operator + │ + ▼ + authenticated ingress adapters + │ + ▼ + durable journal + stable event identity + │ + ▼ partition by (chain_id, e3_id) + per-E3 workflow actor → pure reducer + │ + ▼ + committed effect intents + │ + ▼ + bounded effect runners + │ + ▼ + durable, correlated results ─────► workflow actor +``` -### lib.rs is wiring only +Delivery is at-least-once. Exactly-once execution is not assumed. Correctness comes from stable +identity, idempotent transitions, effect deduplication, and on-chain/read-before-write guards. -```rust -mod actors; -mod domain; -mod repo; -pub mod ext; +For every correctness-relevant step: -pub use actors::*; -pub use domain::*; -pub use repo::*; -``` +1. Validate and deduplicate the input. +2. Reduce it to a new state plus intents. +3. Atomically commit the state transition and/or an outbox record before dispatch. +4. Acknowledge the input only after that commit succeeds. +5. Execute intents outside the actor's critical section. +6. Persist the correlated result before it can unlock the next transition. +7. Retry according to policy until success, terminal failure, cancellation, or persisted deadline. -`lib.rs` must never contain logic or type definitions. If you find yourself adding a `struct` or -`fn` to `lib.rs`, it belongs in a module. +If the current storage abstraction cannot atomically commit a snapshot and outbox entry, record the +intent itself as the durable source of truth and make replay re-derive the state. Never mutate memory +and then rely on a fire-and-forget persistence message as proof of durability. -### Preserving public API across refactors +## State Classification -When moving a type into `domain/`, re-export it from the actor file if external crates referenced -the old path, e.g.: +Every state field is classified during review: -```rust -// in actors/publickey_aggregator.rs -pub use crate::domain::publickey_aggregation::PublicKeyAggregatorState; -``` +| Class | Meaning | Requirement | +| --- | --- | --- | +| Durable | Losing it can change outcome or stall progress | Persist with a versioned schema before acknowledgement | +| Derivable | Reconstructible from authoritative durable facts | Document the source and test reconstruction | +| Ephemeral | Cache/telemetry only; safe to lose | Bound it and make loss behavior explicit | -The public API (`e3_::SomeType`) must stay byte-for-byte identical so downstream crates don't -break. +Pending proof bundles, decrypted-share progress, accusation votes/timeouts, retry state, active +aggregator designation, deadlines, and undispatched external effects are durable unless a stronger +authoritative source can deterministically recreate them. ---- +Snapshots are optimization checkpoints, not a second authority. Event replay from an earlier +checkpoint and snapshot hydration at the same logical point must produce equivalent workflow state +and pending intents. -## 3. Where to Make Specific Modifications +## Event and Message Taxonomy -| You want to… | Edit here | -| --------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | -| Add/change a protocol state transition or validation rule | `domain/.rs` (a pure method + a decision enum variant) | -| React to a new event type | The actor's `Handler` match arm → delegate to a service method | -| Add a new persisted state | New `Repository` factory in `repo.rs` + a `StoreKeys::()` in `crates/events/src/store_keys.rs` | -| Add a new cross-actor event | New variant in `InterfoldEventData` + struct in `crates/events/src/interfold_event/` + `EventType` entry | -| Add a new actor | New file in `actors/`, register in `actors/mod.rs`, give it an `attach()` ctor, subscribe it on the bus | -| Add a timeout/timer | Schedule in the actor via `ctx.run_later`; compute the _policy_ (when/why) in a pure service | -| Change DKG/proof/aggregation math | The relevant `domain/` service; update `agent/flow-trace/04_*.md` same change | -| Change committee/sortition selection | `crates/sortition/src/domain/`; update flow-trace `03_*.md` | -| Track E3 lifecycle stage | `crates/request/src/domain/lifecycle.rs` (pure) + `actors/lifecycle_coordinator.rs` (thin) | +Names must reflect semantics. Persisting all messages under one `Event` label hides important +reliability differences. ---- +| Kind | Meaning | Examples | Default durability | +| --- | --- | --- | --- | +| Fact | Something already happened | `CommitteeFinalized`, `ProofVerified`, `CiphertextPublished` | Durable | +| Intent | Work that must be attempted | `GenerateC5`, `PublishCommittee`, `ScheduleDeadline` | Durable outbox | +| Result | Outcome of an intent | `C5Generated`, `PublishConfirmed`, `ComputeFailed` | Durable | +| Query | Request for current information | health/status/repository lookup | Ephemeral | +| Infrastructure signal | Runtime lifecycle | `EffectsEnabled`, readiness, shutdown | Durable only when needed for recovery/audit | -## 4. Data Management Through the System +Legacy names may be retained for wire/schema compatibility, but their semantic kind must be +documented. For example, `ComputeRequest` is an intent even if its historical Rust type is stored in +the event journal. -### 4.1 The persistence stack +All durable envelopes carry schema version, event ID, causation ID, origin ID, chain ID where +applicable, aggregate/E3 key, source, and timestamp/watermark metadata. Received network events keep +the sender's stable identity; local transport metadata must not accidentally create a new logical +event on every replay. -``` -Service state (plain struct, Serialize + Deserialize + Clone) - �packaged in⌄ -Persistable // crates/data — in-memory value + durable mirror - ⥥ created by⌄ -Repository // a typed, scoped handle to the KV store - ⥥ produced by⌄ -Repositories factory // store.repositories() → all repo factory traits - ⥥ keyed by⌄ -StoreKeys::xxx() // canonical key strings, ALL defined in one file - ⥥ backed by⌄ -DataStore → InMemStore | SledStore (KV); EventStore (append-only log) -``` +## Routing, Backpressure, and Ordering -### 4.2 Rules for persisted state - -- **Every persisted type is `Serialize + DeserializeOwned + Clone + Send + Sync`.** This is the - `PersistableData` bound. -- **All store keys live in `crates/events/src/store_keys.rs`.** Never inline a raw key string - anywhere else. Add a `StoreKeys::()` method. -- **Define a repository factory trait per crate** in `repo.rs`: - ```rust - pub trait E3LifecycleRepositoryFactory { - fn e3_lifecycle(&self) -> Repository>; - } - impl E3LifecycleRepositoryFactory for Repositories { - fn e3_lifecycle(&self) -> Repository> { - Repository::new(self.store.scope(StoreKeys::e3_lifecycle())) - } - } - ``` -- **Load with a default** at actor `attach()` time: - ```rust - let store = repo.load_or_default(HashMap::new()).await?; - ``` -- **Mutate through `Persistable`**, never bypass it: - - `try_mutate(&ctx, |state| Ok(new_state))` — mutate _with_ an event context so the write is - causally ordered (preferred inside event handlers). - - `try_mutate_without_context(|state| ...)` — only for setup/bootstrapping. - - `set(value)` — overwrite (used by snapshot mirrors like the lifecycle map). - - `clear()` — delete the key. - - `get()` → `Option`; `try_get()` → `Result`. - -### 4.3 EventContext: causal ordering - -Events carry an `EventContext` (`ec`). When persisting in response to an event, thread -that `ec` through `try_mutate(&ec, ...)`. This preserves the hybrid-logical-clock ordering used by -sync/replay. **Do not invent a fresh context** for a write that is caused by an inbound event. - -### 4.4 Snapshot vs. EventStore — two persistence mechanisms - -1. **EventStore** (append-only): every `InterfoldEvent` is logged. On restart the log is - **replayed** (with side-effects disabled) to rebuild state. -2. **Snapshot / Repository KV**: actors persist their current state so replay can start from the - last snapshot instead of genesis. - -Both must agree. A service rebuilt from replay must reach the _same_ state as one restored from a -snapshot. This is why services are pure and transitions monotonic — see §6. - -### 4.5 Upgrade safety — never break on-disk formats - -- Persisted formats are **bincode**. Changing field order/type of a persisted struct breaks - deserialization of existing nodes' data on upgrade. -- When you must evolve a persisted type, keep it backward compatible (additive optional fields, - versioned enums) and **preserve the byte layout of legacy data**. Example: `InMemKvStore::dump()` - still serializes a `BTreeMap` for byte-identical compatibility with pre-HAMT dumps. -- Add a round-trip / legacy-format test when touching a persisted type. - ---- - -## 5. Events & The Bus - -- `EventBus` is the single pub/sub fabric. Actors subscribe via - `bus.subscribe(EventType::X, addr.into())` or `bus.subscribe_all(&[...], addr)`. -- `InterfoldEvent` wraps `InterfoldEventData` (the variant enum) + an `EventContext`. Use - `msg.into_components()` to get `(data, ec)`. -- **Events are facts, not commands.** They describe what happened. An actor decides for itself how - to react. Do not use events as RPC. -- `EffectsEnabled` gates side-effecting work so that historical replay does not re-trigger on-chain - submissions. Subscribe effectful behavior behind it (see sortition's `E3Requested` gating). -- Errors go on the bus too: `bus.err(EType::, anyhow!(...))`. Use a non-blocking error - for recoverable issues; never `panic!` in a handler. - -### Adding an event - -1. Add struct in `crates/events/src/interfold_event/.rs` (derive - `Message, Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize`). -2. Add a variant to `InterfoldEventData` and an `EventType` entry in `interfold_event/mod.rs`. -3. Re-export from `mod.rs`. -4. If it carries an E3, ensure `get_e3_id()` returns it. -5. Update the relevant flow-trace doc. - ---- - -## 6. The E3 Lifecycle Coordinator (single source of truth for stage) - -The node is **choreographed** — no component "drives" the protocol. To still have one durable answer -to "what stage is each E3 at?", `e3-request` provides: - -- `domain/lifecycle.rs` — pure `E3LifecycleService { stages: HashMap }`. - - `observe(&InterfoldEventData) -> LifecycleDecision` - - Stage advance is **monotonic / forward-only** (ranked). - - Terminal stages (`Complete`, `Failed`) are **frozen**. - - Out-of-order earlier-stage events return `Regressed` and are ignored. -- `actors/lifecycle_coordinator.rs` — thin `E3LifecycleCoordinator` actor that loads the persisted - map, subscribes to lifecycle events + `Shutdown`, persists on `Advanced`/`Terminal`, logs active - E3s on shutdown. - -**The coordinator is ADDITIVE.** It observes and records. It **must never** emit protocol events or -drive subsystems — doing so would break the choreography and create duplicate effects. Stage -mapping: - -| Event | Stage | -| ------------------------------------------------------------------------ | --------------------------- | -| `E3Requested` | `Requested` | -| `CommitteePublished` / `CommitteeFinalized` | `CommitteeFinalized` | -| `PublicKeyAggregated` | `KeyPublished` | -| `CiphertextOutputPublished` | `CiphertextReady` | -| `PlaintextAggregated` / `PlaintextOutputPublished` / `E3RequestComplete` | `Complete` | -| `E3Failed` | `Failed` | -| `E3StageChanged` | `new_stage` (authoritative) | - ---- - -## 7. Decision Enums — the service↔actor contract - -Services return their decisions as explicit enums; actors `match` and apply them. This keeps the I/O -boundary visible and testable. - -Conventions: - -- Name outcomes by what the actor must do, not by internal state: - `{ Ignored, Pending, Completed(..) }`, - `RoutingDecision { Broadcast, Ignore, Process { post_forward }, AlreadyCompleted }`, - `Vec`, etc. -- Prefer returning a `Vec` when one event causes several effects. -- Never return raw booleans for multi-way decisions — use an enum. -- The actor's job after `match` is strictly: persist, publish, schedule, forward. - ---- - -## 8. Testing Conventions - -- **Unit tests live next to the service** they cover, in `#[cfg(test)] mod tests` inside the - `domain/` file. They are synchronous, no actix, no bus, no store. -- Use property/oracle tests for data structures (e.g. HAMT vs `BTreeMap` over thousands of random - ops). -- Actor-level tests are only for genuinely actor-specific wiring; if the bus setup is heavy and the - logic is already covered by the service tests, do not duplicate it at the actor level. -- When you move a test during a refactor, **move it next to the new home** of the logic — never - leave orphaned tests behind. -- Verify against **real APIs**. Do not fabricate constructors/`Default` impls in tests; check the - actual type first. - -### Verification commands (run before claiming done) - -```bash -cargo build --workspace -cargo clippy -p --all-targets # clean on the crate's own src/ -cargo test -p -# Full gates: -pnpm rust:test -pnpm test:integration -pnpm check:committee # circuit config invariant -``` +- Protocol work is partitioned by `(chain_id, e3_id)` so one expensive or blocked E3 cannot pause + unrelated E3s. +- Ordering is guaranteed within a partition. Global total ordering is used only where a documented + invariant requires it. +- Correctness-critical sends are acknowledged and bounded by timeout. A failed send is retried or + escalated; it is never only logged. +- Buffers are bounded by both item count and bytes. Overflow has an explicit policy and metric. +- Replay uses bounded paging/merge and the same acknowledgement semantics as live delivery. +- Queries and telemetry may use lossy delivery only when callers can distinguish loss/unavailability. + +`do_send` is allowed for best-effort telemetry. It is not allowed for state persistence, workflow +progression, timers, proof/results, cleanup, network publication, or external transaction intents. + +## Timers and Deadlines + +Persist the absolute protocol deadline and timer purpose, not only an in-memory Actix handle. On +restart, compare the persisted deadline with an injected clock and deterministically emit either a +new timer intent or the overdue input. Timer cancellation is itself part of the workflow transition. + +Staggered submitters use a stable rank and persisted attempt state. Restarting must not reset a +fallback delay in a way that suppresses the only remaining submitter. + +## Choreography and Local Orchestration + +Interfold remains choreographed across nodes and contracts: no node is the global coordinator. +Inside one node, a per-E3 workflow is explicitly orchestrated so its durable state answers: + +- what facts have been accepted; +- which stage and canonical participants apply; +- which operations are pending, running, succeeded, or terminally failed; +- which deadlines remain; +- which actor/worker owns each in-flight operation. + +The `E3LifecycleCoordinator` is a projection, not the source of truth. Authoritative stage comes +from canonical chain facts plus durable local workflow facts. A projection may be deleted and +rebuilt without changing execution. + +## Schema Evolution + +Rust type compatibility is not a storage migration strategy. Every durable event, snapshot, and +outbox payload has an explicit schema version and decoder policy. + +- Adding/removing/reordering a field requires a compatibility test against checked-in fixtures. +- Bincode payloads are never assumed self-describing or forward compatible. +- A version mismatch either runs a tested migration or fails startup with an actionable error. +- Migrations are restartable and do not destroy the previous data until the replacement is verified. +- Wire compatibility and storage compatibility are reviewed separately. + +## Testing Requirements + +Domain and workflow tests are actor-free and deterministic. They cover: + +- every legal and illegal transition; +- duplicates, reordering, missing parties, expulsion, and threshold boundaries; +- canonical ordering and proof multiplicities for every supported preset; +- intent idempotency keys and terminal failure mapping. + +Runtime tests cover: + +- mailbox saturation, unavailable recipients, and bounded timeouts; +- worker failure/cancellation and actor restart; +- duplicate fact/result delivery; +- shutdown barriers and effect gating. + +Recovery tests use a crash matrix around each effect: + +1. before transition commit; +2. after commit but before dispatch; +3. while the effect is running; +4. after external success but before result commit; +5. after result commit but before the next input. + +Each case must converge to the same state and external outcome as uninterrupted execution. Snapshot +hydration and full replay must produce equivalent state plus pending intents. + +Integration tests assert end-to-end protocol behavior. Long cryptographic tests run after fast +domain, workflow, crate, and workspace checks have passed. + +The recursive `node_fold_correlated_sparse_self_slot_proves_and_verifies` test and the full +`test_trbfv_actor` flow belong to the slow lane. Debug builds may spend minutes in real proof/FHE +work and may emit a "running for over 60 seconds" progress warning. That warning is not a failure; +the test harness exit status is authoritative. Keep these tests enabled, run them last, and give CI +an explicit timeout based on measured debug-runtime headroom rather than weakening their coverage. + +## Code Shape and Review Heuristics + +Files and structs are organized around one reason to change. Roughly 300 lines is a review trigger, +not a mechanical limit: generated bindings, tables, and cohesive algorithms may be longer. A large +file must not combine unrelated message definitions, domain rules, persistence, actor handlers, +effect execution, and tests. + +A struct is likely a god object when it owns several independent lifecycles, mixes durable and +ephemeral state without classification, or requires most dependencies for only a few handlers. +Extract behavior by responsibility: + +1. pure domain rule or value; +2. workflow state and reducer; +3. effect port/runner; +4. repository/codec; +5. actor façade and message routing; +6. test support. + +Do not hide coupling by splitting one `impl` into arbitrary files while keeping the same god struct. +The goal is smaller ownership and explicit contracts, not a lower line count alone. + +## Migration Rules + +Refactor one recoverable protocol slice at a time: + +1. Lock current behavior with protocol and compatibility tests. +2. Classify its messages and state. +3. Extract pure domain decisions. +4. Introduce workflow state, inputs, intents, and stable operation IDs. +5. Add durable dispatch/result handling. +6. Move concrete I/O into bounded adapters. +7. Add crash/replay equivalence tests. +8. Remove the legacy path only after both paths agree. +9. Update `CRATES_ARCHITECTURE.md` and the relevant flow trace in the same change. + +Temporary bridges are permitted when named, tested, and tracked for removal. New code must not add +another unacknowledged correctness edge merely because a neighboring legacy path still has one. + +## Definition of Done for an Actor Refactor + +An actor is considered architecturally thin when: + +- its protocol decisions can be tested without Actix; +- its durable state and derivable/ephemeral fields are explicit; +- its handlers translate inputs, invoke reducers, commit, and dispatch intents; +- heavy work and external I/O run behind bounded effect interfaces; +- every critical send and persistence step has success/failure semantics; +- restart restores pending deadlines and effects; +- duplicate and out-of-order delivery is deterministic; +- supervision, cancellation, cleanup, readiness, and metrics are defined; +- documentation and flow traces describe the implemented behavior. -The workspace baseline is **not** clippy-clean; only enforce clippy-clean on the crate you touched -(its own `src/`). - ---- - -## 9. Dos and Don'ts - -### Do - -- Put every state transition, validation, and computation in a pure `domain/` service with unit - tests. -- Keep `lib.rs` to module decls + re-exports. -- Define one struct per file; name the file after the struct. -- Route all persistence through `Repository`/`Persistable` and `StoreKeys`. -- Thread the inbound `EventContext` into `try_mutate`. -- Make stage/state advances monotonic and replay-safe. -- Preserve on-disk (bincode) formats; add legacy round-trip tests. -- Update the matching `agent/flow-trace/*.md` in the same change when behavior, signatures, events, - CLI, timeouts, or proof steps change. -- Gate side-effects behind `EffectsEnabled` so replay is safe. -- Delete old files when refactoring; no duplicates left behind. - -### Don't - -- Put business logic in a `Handler` (no validation/crypto/aggregation/math). -- Add types or functions to `lib.rs`. -- Inline raw store-key strings outside `store_keys.rs`. -- Invent a fresh `EventContext` for an event-caused write. -- Emit protocol events from the lifecycle coordinator (it is observe-only). -- Use events as commands/RPC. -- `panic!`/`unwrap()` in handlers — surface errors on the bus. -- Change a persisted struct's field order/type without an upgrade-safe path. -- Block the actor thread on long sync work (offload; keep handlers fast). -- Rename crates and move logic in the same change (renames are a separate mechanical PR). -- Hand-edit the three circuit config files — use `pnpm build:circuits`. - ---- - -## 10. Crate Map (subsystems) - -| Subsystem | Crates | -| ------------ | ---------------------------------------------------------------------------------------------------------- | -| Protocol | `aggregator`, `keyshare`, `sortition`, `slashing`, `request`, `committee-hash` | -| FHE / crypto | `fhe`, `trbfv`, `bfv-client`, `multithread`, `crypto`, `fhe-params`, `polynomial`, `parity-matrix`, `safe` | -| ZK | `zk-prover`, `zk-helpers`, `compute-provider` | -| EVM | `evm`, `evm-helpers`, `indexer` | -| Runtime | `events`, `data`, `sync`, `net`, `config`, `logger`, `fs`, `hamt` | -| Node | `ciphernode-builder`, `entrypoint`, `cli`, `daemon-server`, `console`, `init`, `dashboard` | -| Clients | `sdk`, `wasm`, `program-server` | -| Tooling | `interfoldup`, `utils`, `utils-derive`, `test-helpers`, `support`, `scripts`, `tests` | - -All crates use the `e3-` package prefix except `interfoldup`. - ---- - -## 11. Refactor Pattern Recap (the proven playbook) - -When asked to extract logic from an actor: - -1. Read the actor file(s) fully; identify embedded logic. -2. Create `domain/.rs` with a plain struct + decision enum; move domain types + transition - fns; add `#[cfg(test)]` tests. -3. Register in `domain/mod.rs`; re-export from `lib.rs`. -4. Thin each `Handler` to: destructure → call service → apply decision. -5. Re-export moved types from the actor file to preserve public paths. -6. `cargo test -p ` + `cargo clippy -p --all-targets`; build downstream consumers; - `cargo build --workspace`. -7. Update flow-trace only if observable behavior/signatures/events changed. -8. Delete the old flat file. No duplicates. +The actor model is therefore retained, but actors cease to be containers for the whole protocol. +They become reliable runtime shells around deterministic, recoverable workflows. diff --git a/agent/CRATES_ARCHITECTURE.md b/agent/CRATES_ARCHITECTURE.md index 20f892dd92..d621151213 100644 --- a/agent/CRATES_ARCHITECTURE.md +++ b/agent/CRATES_ARCHITECTURE.md @@ -4,7 +4,8 @@ This document describes the implementation in `crates/`. It is intentionally code-facing: names in the diagrams are crate, module, actor, message, or durable repository names that can be searched in the workspace. Read it alongside the prescriptive [`ARCHITECTURE.md`](ARCHITECTURE.md) contribution guide and -[`RULES.md`](RULES.md). +[`RULES.md`](RULES.md). The actor-by-actor refactor findings are summarized in +[`ACTOR_AUDIT.md`](ACTOR_AUDIT.md). ## Dependency layers @@ -175,32 +176,34 @@ Protocol workflow crates also depend directly on Actix and concrete repositories These are real constraints in the current code and are not papered over with empty port traits. -The resulting domain/application/adapter placement is module-level rather than a -clean crate boundary: +The resulting workflow/actor/effect separation is module-level rather than a clean +crate boundary. On disk, those roles are grouped by protocol capability; the labels in +this diagram describe responsibilities, not top-level source directories: ```mermaid flowchart LR - Composition[CiphernodeBuilder and entrypoint composition] --> App[Actix-backed protocol workflows] + Composition[CiphernodeBuilder and entrypoint composition] --> Actors[Actix runtime boundaries] Composition --> Adapters[concrete EVM, libp2p, storage, and proof adapters] Composition --> Infra[Actix, RPC, libp2p, bb, stores, and task pools] - App --> Domain[deterministic domain modules and invariant types] - App --> Adapters + Actors --> Workflow[deterministic workflow state and decisions] + Actors --> Adapters + Workflow --> Domain[protocol values, validation, and invariants] Adapters --> Infra Domain --> Shared[e3-events payloads and transport types] - App --> Shared + Workflow --> Shared + Actors --> Shared Adapters --> Shared classDef debt fill:#fff1f0,stroke:#cf222e,color:#82071e - class App,Shared debt + class Actors,Shared debt ``` Pure decision modules have no actor runtime (for example lifecycle transitions, sync -planning, network buffer decisions, document validation, accusation voting, and -aggregation state machines). Application coordination is still predominantly actor -handlers rather than independent service objects. Adapters are concrete and are wired -centrally by `CiphernodeBuilder`. Arrows in this diagram point from a consumer to what -it uses: domain code does not depend on the composition root, while application actors -still depend directly on concrete adapters in several crates. +planning, network buffer decisions, document validation, accusation voting, proof +dispatch/verification, and aggregation state machines). Adapters are concrete and are +wired centrally by `CiphernodeBuilder`. Arrows in this diagram point from a consumer to +what it uses: domain code does not depend on the composition root, while actors still +depend directly on concrete adapters in several crates. ## Ciphernode construction and startup @@ -278,20 +281,51 @@ flowchart LR ``` Actors own scheduling, mailbox ordering, subscriptions, and lifecycle. Deterministic -decisions that have already been separated live in modules such as -`request::domain::lifecycle`, `sync::domain::sync_planner`, -`net::domain::net_buffer`, `slashing::domain::accusation_voting`, and the typed -aggregation state machines. The audit identifies handlers that still combine policy -with transport. The normal live path shown above is durable-before-dispatch, but it is -not fully backpressured: several `BusHandle`, `Sequencer`, `EventStoreRouter`, and -snapshot edges still use Actix `do_send`, which bypasses mailbox capacity. That debt is -called out explicitly below rather than representing the pipeline as end-to-end -bounded. ZK proof actors are composition-scoped EventBus subscribers. Threshold +decisions are physically co-located with their capabilities, including +`request/src/lifecycle/workflow.rs`, `sync/src/sync/workflow.rs`, +`net/src/event_buffer/workflow.rs`, `slashing/src/accusation_voting/workflow.rs`, +`zk-prover/src/{proof_request,share_verification}/workflow.rs`, and the typed aggregation +workflows. Compatibility views such as `domain.rs` and `workflow.rs` preserve established +Rust module paths while that migration settles; they contain declarations, not business logic. +Several `BusHandle`, `Sequencer`, `EventStoreRouter`, and snapshot edges still +contain Actix `do_send`, so the pipeline is not end-to-end backpressured. That debt is +called out explicitly below. ZK proof actors are +composition-scoped EventBus subscribers. Threshold keyshare and public-key/plaintext aggregation actors are request-scoped recipients created by `E3Router` extensions and reached through `E3Context`; they are not direct EventBus subscribers. Per-E3 accusation and consistency actors are context-owned but also install direct subscriptions for the proof and slash events they consume. +### Capability refactor map + +Every production actor was inventoried during the architecture refactor. Thinness is +judged by ownership, not raw line count; roughly 300 production lines is a review +trigger. The actor-bearing crates now use one filesystem rule: capability directories +contain predictable role files such as `actor.rs`, `handlers.rs`, `state.rs`, +`workflow.rs`, `effects.rs`, and adjacent tests. A role becomes a subdirectory only +when it has several independent concerns, and those files receive semantic operation +names rather than circuit-stage labels. No `src/actors/`, `src/domain/`, +`src/workflow/`, `src/adapters/`, or `src/runtime/` layer directory remains in these +crates. + +| Crate | Capability directories | Boundary after refactor | +| --- | --- | --- | +| `e3-aggregator` | `committee_finalization`, `public_key_aggregation`, `plaintext_aggregation` | Request-local actor shells own timing and routing; workflows own aggregation decisions and semantic effect files own proof/publication work. | +| `e3-keyshare` | `threshold_keyshare` | One request-local mailbox coordinates DKG; collectors, state, pure key/share calculations, handlers, and effect operations are co-located by capability. | +| `e3-zk-prover` | `proof_request`, `proof_verification`, `share_verification`, `node_proof_aggregation`, `commitment_links` | Proof mailboxes dispatch work; workflows and commitment-link modules own pure decisions, while semantic effect files own circuit requests and publication. | +| `e3-slashing` | `accusation_voting`, `commitment_consistency` | Actors own timers and message routing; workflow files own admission, verification, voting, quorum, and commitment decisions. | +| `e3-sortition` | `sortition`, `ciphernode_selection` | Actors own chain/request routing and cache lifecycle; selection backends, ticket rules, and registry decisions sit beside them. | +| `e3-net` | `event_buffer`, `event_conversion`, `event_translation`, `network_sync`, `document_publishing` | Mailboxes own transport ordering and lifecycle; workflow/model files own decisions and effects own DHT, gossip, and history I/O. | +| `e3-evm` | `chain_gateway`, `chain_reader`, `event_decoding`, registry/interfold/slashing read and write capabilities, `log_fetching` | Per-chain mailboxes own concurrency; provider recovery, log fetching, transaction preflight, and submission live with the chain capability they serve. | +| `e3-request` | `routing`, `lifecycle` | Context routing and lifecycle mailboxes call deterministic workflows; snapshot/context construction is co-located with routing. | +| `e3-sync` | `sync` | No Actix actor: an acknowledged startup/replay service contains its state, plan, preflight, history collection, and tests in one capability. | + +The remaining large non-actor files are not automatically actor violations. Generated +contract bindings and cohesive circuit/FHE algorithms are reviewed by their own +complexity and test boundaries. Composition roots such as `CiphernodeBuilder`, and +infrastructure coordinators such as `NetInterface`, remain separate follow-up targets; +splitting them mechanically would not make protocol actors thinner. + ## Event ingestion, persistence, replay, and synchronization ```mermaid diff --git a/agent/flow-trace/04_DKG_AND_COMPUTATION.md b/agent/flow-trace/04_DKG_AND_COMPUTATION.md index 20b83276df..bbac1c4da4 100644 --- a/agent/flow-trace/04_DKG_AND_COMPUTATION.md +++ b/agent/flow-trace/04_DKG_AND_COMPUTATION.md @@ -54,7 +54,11 @@ CiphernodeSelected event arrives at ThresholdKeyshare ### Step 2: C0 Proof Generation → EncryptionKeyCreated -**Actor:** `ProofRequestActor` (`crates/zk-prover/src/actors/proof_request.rs`) +**Actor:** `ProofRequestActor` (`crates/zk-prover/src/proof_request/actor.rs`) + +**Deterministic workflow:** pending proof state and canonical dispatch sequence planning live in +`crates/zk-prover/src/proof_request/{state,workflow,transitions}.rs`; `handlers.rs` owns mailbox +entry and the semantic files below `effects/` own correlation, signing, and compute dispatch. ``` ProofRequestActor receives EncryptionKeyPending @@ -362,12 +366,17 @@ ThresholdShareCollector waits for ThresholdShareCreated from ALL N parties ### Step 6a: C2/C3 Share Proof Verification -**Actor:** `ShareVerificationActor` (`crates/zk-prover/src/actors/share_verification.rs`) +**Actor:** `ShareVerificationActor` (`crates/zk-prover/src/share_verification/actor.rs`) + +**Deterministic workflow:** signature/slot validation, replay normalization, consistency filtering, +and result tallying live in `crates/zk-prover/src/share_verification/{state,workflow}.rs`. The +capability's `handlers.rs` owns mailbox entry and its semantic effect files own correlation state +and publish returned decisions. ``` ShareVerificationActor receives ShareVerificationDispatched(kind=ShareProofs) │ -├─ PHASE 1: Lightweight ECDSA Validation (inline): +├─ PHASE 1: Lightweight ECDSA Validation (workflow service): │ │ │ ├─ For EACH party's proofs: │ │ ├─ Verify e3_id matches diff --git a/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md b/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md index 7718d8d8d7..cc1e0a6056 100644 --- a/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md +++ b/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md @@ -256,7 +256,12 @@ Same scenario as above, then 2 nodes are slashed for 300,000 each: ### Off-Chain Fault Attribution: AccusationManager -**Actor:** `AccusationManager` (`crates/zk-prover/src/actors/accusation_manager.rs`) +**Actor:** `AccusationManager` (`crates/slashing/src/accusation_voting/actor.rs`; the +`zk-prover` path is a compatibility re-export) + +**Deterministic workflow:** `crates/slashing/src/accusation_voting/workflow.rs` owns deadlines, +EIP-712 digests, admission, vote/quorum decisions, and re-verification state. The actor owns timers +and executes the workflow's returned `VoteAction`s. The AccusationManager is a per-E3 ephemeral actor created when `SortitionCommitteeFinalized` (the `ICiphernodeRegistry` event) fires. It bridges proof verification failures to on-chain slashing diff --git a/crates/aggregator/src/actors/mod.rs b/crates/aggregator/src/actors.rs similarity index 57% rename from crates/aggregator/src/actors/mod.rs rename to crates/aggregator/src/actors.rs index 0fb200d2dc..ded2eb42ad 100644 --- a/crates/aggregator/src/actors/mod.rs +++ b/crates/aggregator/src/actors.rs @@ -4,13 +4,19 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -//! Thin actix actor shells. These types handle message passing only and -//! delegate all business logic to the pure services in [`crate::domain`]. +//! Compatibility view of the thin actors stored with their capabilities. +//! +//! New implementation belongs in the referenced capability directory, not here. +#[path = "committee_finalization/actor.rs"] mod committee_finalizer; +#[path = "plaintext_aggregation/decryption_share_buffer.rs"] mod decryptionshare_created_buffer; +#[path = "public_key_aggregation/keyshare_buffer.rs"] mod keyshare_created_filter_buffer; +#[path = "public_key_aggregation/actor.rs"] mod publickey_aggregator; +#[path = "plaintext_aggregation/actor.rs"] mod threshold_plaintext_aggregator; pub use committee_finalizer::*; diff --git a/crates/aggregator/src/actors/publickey_aggregator.rs b/crates/aggregator/src/actors/publickey_aggregator.rs deleted file mode 100644 index b2a8e060b3..0000000000 --- a/crates/aggregator/src/actors/publickey_aggregator.rs +++ /dev/null @@ -1,1961 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::domain::committee::committee_addresses_in_party_order; -use crate::domain::publickey_aggregation::{ - check_c1_keyshare_commitments, extract_pk_commitment, verify_dkg_fold_attestation, C1Dispatch, - HonestSelection, PublicKeyAggregation, -}; -use actix::prelude::*; -use anyhow::Result; -use e3_data::Persistable; -use e3_events::{ - prelude::*, BusHandle, ComputeRequest, ComputeRequestError, ComputeResponse, - ComputeResponseKind, CorrelationId, DKGRecursiveAggregationComplete, Die, - DkgAggregationRequest, E3Failed, E3Stage, E3id, EventContext, FailureReason, InterfoldEvent, - InterfoldEventData, KeyshareCreated, NodesFoldStepRequest, OrderedSet, - PkAggregationProofPending, PkAggregationProofRequest, PkAggregationProofSigned, Proof, - ProofType, PublicKeyAggregated, Sequenced, ShareVerificationComplete, - ShareVerificationDispatched, SignedProofFailed, SignedProofPayload, TypedEvent, - VerificationKind, ZkRequest, ZkResponse, -}; -use e3_events::{trap, EType}; -use e3_fhe::{Fhe, GetAggregatePublicKey}; -use e3_fhe_params::BfvPreset; -use e3_utils::NotifySync; -use e3_utils::{ArcBytes, MAILBOX_LIMIT}; -use e3_zk_helpers::CiphernodesCommitteeSize; -use std::collections::HashMap; -use std::sync::Arc; -use tracing::{error, info, warn}; - -// Public-key aggregation state machine + pure transition logic now live in -// `crate::domain::publickey_aggregation`; re-exported here to preserve the public path -// `e3_aggregator::publickey_aggregator::PublicKeyAggregatorState`. -pub use crate::domain::publickey_aggregation::PublicKeyAggregatorState; - -pub struct PublicKeyAggregator { - fhe: Arc, - bus: BusHandle, - e3_id: E3id, - state: Persistable, - params_preset: BfvPreset, - committee_size: CiphernodesCommitteeSize, - /// DKG recursive aggregation events received before entering GeneratingC5Proof. - early_dkg_proofs: Vec>, -} - -pub struct PublicKeyAggregatorParams { - pub fhe: Arc, - pub bus: BusHandle, - pub e3_id: E3id, - pub params_preset: BfvPreset, - pub committee_size: CiphernodesCommitteeSize, -} - -/// Aggregate PublicKey for a committee of nodes. This actor listens for KeyshareCreated events -/// around a particular e3_id, verifies C1 proofs, aggregates the public key, generates a C5 -/// proof of correct aggregation, and broadcasts a PublicKeyAggregated event on the event bus. -impl PublicKeyAggregator { - pub fn new( - params: PublicKeyAggregatorParams, - state: Persistable, - ) -> Self { - PublicKeyAggregator { - fhe: params.fhe, - bus: params.bus, - e3_id: params.e3_id, - state, - params_preset: params.params_preset, - committee_size: params.committee_size, - early_dkg_proofs: Vec::new(), - } - } - - pub fn add_keyshare( - &mut self, - keyshare: ArcBytes, - node: String, - party_id: u64, - c1_proof: Option, - ec: &EventContext, - ) -> Result<()> { - self.state.try_mutate(ec, |state| { - PublicKeyAggregation::add_keyshare( - state, - keyshare.clone(), - node.clone(), - party_id, - c1_proof.clone(), - ) - }) - } - - fn dispatch_c1_verification( - &mut self, - submission_order: &[(u64, String, ArcBytes)], - c1_proofs: &[Option], - ec: EventContext, - ) -> Result<()> { - let C1Dispatch { - party_proofs, - no_proof_parties, - } = PublicKeyAggregation::plan_c1_dispatch(submission_order, c1_proofs); - - // Store no-proof parties in state for the response handler - if !no_proof_parties.is_empty() { - self.state.try_mutate(&ec, |mut state| { - if let PublicKeyAggregatorState::VerifyingC1 { - no_proof_parties: ref mut stored, - .. - } = state - { - *stored = no_proof_parties.clone(); - } - Ok(state) - })?; - } - - if party_proofs.is_empty() { - return Err(anyhow::anyhow!( - "No C1 proofs to verify — all keyshares must include a signed C1 proof" - )); - } - - info!( - "Dispatching C1 proof verification for {} parties ({} missing proofs)", - party_proofs.len(), - no_proof_parties.len() - ); - - self.bus.publish( - ShareVerificationDispatched { - e3_id: self.e3_id.clone(), - kind: VerificationKind::PkGenerationProofs, - share_proofs: party_proofs, - decryption_proofs: vec![], - pre_dishonest: no_proof_parties.into_iter().collect(), - params_preset: self.params_preset, - committee_size: self.committee_size, - }, - ec, - )?; - Ok(()) - } - - fn handle_c1_verification_complete( - &mut self, - msg: TypedEvent, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - - if msg.kind != VerificationKind::PkGenerationProofs { - return Ok(()); - } - - if msg.e3_id != self.e3_id { - return Ok(()); - } - - let PublicKeyAggregatorState::VerifyingC1 { - submission_order, - threshold_m, - circuit_committee_n, - circuit_committee_h, - c1_proofs, - .. - } = self - .state - .get() - .ok_or_else(|| anyhow::anyhow!("Expected VerifyingC1 state"))? - else { - return Err(anyhow::anyhow!( - "handle_c1_verification_complete called outside VerifyingC1 state" - )); - }; - - let mut dishonest_parties = msg.dishonest_parties.clone(); - let collected = submission_order.len(); - let circuit_h = circuit_committee_h; - - // Retain full N committee roster (party_id → node address) for the DKG aggregator - // `committee_members` input, which must cover all `topNodes` regardless of honesty. - let full_submission_order: Vec<(u64, String, ArcBytes)> = submission_order.clone(); - - // Filter out parties that failed C1 ZK verification. Keyed by the real - // sortition party_id carried in `submission_order`, not arrival index. - let mut honest_entries: Vec<(u64, String, ArcBytes, Option)> = - submission_order - .into_iter() - .zip(c1_proofs) - .filter(|((pid, _, _), _)| !dishonest_parties.contains(pid)) - .map(|((pid, node, ks), c1)| (pid, node, ks, c1)) - .collect(); - - // Cross-check: verify each party's keyshare matches their C1 pk_commitment. - // Parties that fail are marked dishonest and reported via SignedProofFailed. - let audit = check_c1_keyshare_commitments(&honest_entries, &self.fhe); - for party_id in &audit.missing_proof { - dishonest_parties.insert(*party_id); - } - - // Emit SignedProofFailed for each commitment-mismatched party - for (party_id, signed_proof) in &audit.mismatched { - dishonest_parties.insert(*party_id); - match signed_proof.recover_address() { - Ok(faulting_node) => { - if let Err(e) = self.bus.publish( - SignedProofFailed { - e3_id: self.e3_id.clone(), - faulting_node, - proof_type: ProofType::C1PkGeneration, - signed_payload: signed_proof.clone(), - }, - ec.clone(), - ) { - error!("Failed to publish SignedProofFailed: {e}"); - } - } - Err(e) => warn!( - "Could not recover address from C1 proof for party {}: {e}", - party_id - ), - } - } - - if !audit.mismatched.is_empty() { - warn!( - "C1 commitment mismatch for {} parties — filtering before aggregation", - audit.mismatched.len() - ); - // Re-filter honest_entries after commitment check - honest_entries.retain(|(pid, _, _, _)| !dishonest_parties.contains(pid)); - } - - // Sort, fail-closed below H, cap to the H lowest party_ids, and fail when - // <= threshold_m remain. All pure decision logic lives in the service; the - // actor only publishes E3Failed on the Fail outcome. - let (honest_entries, honest_party_ids) = match PublicKeyAggregation::select_honest_set( - &self.e3_id, - honest_entries, - &dishonest_parties, - circuit_h, - threshold_m, - collected, - ) { - HonestSelection::Fail => { - self.bus.publish( - E3Failed { - e3_id: self.e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DKGInvalidShares, - }, - ec, - )?; - return Ok(()); - } - HonestSelection::Proceed { - honest_entries, - honest_party_ids, - } => (honest_entries, honest_party_ids), - }; - - let (honest_keyshares, honest_nodes): (Vec, Vec) = honest_entries - .iter() - .map(|(_, node, ks, _)| (ks.clone(), node.clone())) - .unzip(); - - debug_assert_eq!( - honest_party_ids.len(), - honest_keyshares.len(), - "honest roster and keyshare payload lengths must match" - ); - - // Synchronous aggregation - info!( - "Aggregating public key from {} honest shares...", - honest_keyshares.len() - ); - let honest_keyshares_set = OrderedSet::from(honest_keyshares.clone()); - let pubkey = self.fhe.get_aggregate_public_key(GetAggregatePublicKey { - keyshares: honest_keyshares_set.clone(), - })?; - - let committee_h = honest_keyshares.len(); - let honest_nodes_set = OrderedSet::from(honest_nodes.clone()); - // Feed keyshares to C5 in ascending party_id order so that - // `c5_public[i]` (pk_commitment of the i-th input keyshare) matches - // party_ids[i] and the row-i node_fold pk bound by dkg_aggregator.nr. - // `honest_keyshares` preserves the submission-index (== party_id) order - // from `honest_entries`; do NOT sort by byte content. - let keyshare_bytes: Vec = honest_keyshares.clone(); - - let pubkey = ArcBytes::from_bytes(&pubkey); - info!("Publishing PkAggregationProofPending for C5 proof generation..."); - self.bus.publish( - PkAggregationProofPending { - e3_id: self.e3_id.clone(), - proof_request: PkAggregationProofRequest { - keyshare_bytes: keyshare_bytes.clone(), - aggregated_pk_bytes: pubkey.clone(), - params_preset: self.params_preset, - // C5 witness uses `committee_h` keyshares; artifact lookup needs canonical (N, H, T). - committee_n: circuit_committee_n, - committee_h, - committee_threshold: threshold_m, - }, - public_key: pubkey.clone(), - nodes: honest_nodes_set.clone(), - }, - ec.clone(), - )?; - - // `party_nodes` covers the FULL registered committee (all N keyshare submitters), - // not just the H honest set. The DKG aggregator circuit binds `committee_members` - // to on-chain `topNodes` which always carries the full committee — so we must keep - // the dishonest addresses available here to build the N-sized address vector. - // `submission_order` here is the unfiltered list captured pre–C1 verification - // (the original `VerifyingC1.submission_order`); `honest_entries` is the H subset. - let party_nodes: HashMap = full_submission_order - .iter() - .map(|(pid, node, _)| (*pid, node.clone())) - .collect(); - - let circuit_committee_h = circuit_h; - self.state.try_mutate(&ec, |_| { - Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key: pubkey.clone(), - keyshare_bytes, - nodes: honest_nodes_set, - party_nodes, - dkg_node_proofs: HashMap::new(), - dkg_fold_attestations: HashMap::new(), - honest_party_ids: honest_party_ids.clone(), - dishonest_parties: dishonest_parties.clone(), - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation: None, - dkg_aggregated_proof: None, - c5_proof_pending: None, - last_ec: Some(ec.clone()), - nodes_fold_accumulator: None, - nodes_fold_completed_slots: 0, - nodes_fold_step_correlation: None, - }) - })?; - - // Replay any DKG proofs that arrived before we entered GeneratingC5Proof. - let early = std::mem::take(&mut self.early_dkg_proofs); - for event in early { - self.handle_dkg_recursive_aggregation_complete(event)?; - } - - self.try_dispatch_dkg_aggregation(&ec)?; - - Ok(()) - } - - fn handle_pk_aggregation_proof_signed( - &mut self, - msg: TypedEvent, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - - if msg.e3_id != self.e3_id { - return Ok(()); - } - - if !matches!( - self.state.get(), - Some(PublicKeyAggregatorState::GeneratingC5Proof { .. }) - ) { - return Err(anyhow::anyhow!( - "handle_pk_aggregation_proof_signed called outside GeneratingC5Proof state" - )); - } - - info!("C5 proof signed — waiting for cross-node DKG fold to complete..."); - - let c5_proof = msg.signed_proof.payload.proof.clone(); - self.state.try_mutate(&ec, |state| { - let PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - .. - } = state - else { - return Ok(state); - }; - Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - c5_proof_pending: Some(c5_proof), - last_ec: Some(ec.clone()), - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - }) - })?; - self.try_publish_complete() - } - - // -- Cross-node DKG proof aggregation -------------------------------------------------- - - fn handle_dkg_recursive_aggregation_complete( - &mut self, - msg: TypedEvent, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - - if msg.e3_id != self.e3_id { - return Ok(()); - } - - let state = self.state.get(); - let Some(PublicKeyAggregatorState::GeneratingC5Proof { - party_nodes, - dkg_node_proofs, - honest_party_ids, - circuit_committee_n, - circuit_committee_h, - .. - }) = state.as_ref() - else { - info!( - "PublicKeyAggregator: early DKG proof from party {} — buffering until GeneratingC5Proof", - msg.party_id - ); - self.early_dkg_proofs.push(TypedEvent::new(msg, ec)); - return Ok(()); - }; - if dkg_node_proofs.contains_key(&msg.party_id) { - warn!( - "Duplicate DKGRecursiveAggregationComplete for party {} — ignoring", - msg.party_id - ); - return Ok(()); - } - - if honest_party_ids.contains(&msg.party_id) { - let Some(expected_node) = party_nodes.get(&msg.party_id) else { - warn!( - party_id = msg.party_id, - "DKG fold from party without registered node address — rejecting" - ); - return Ok(()); - }; - // Proof aggregation OFF: nodes emit `DKGRecursiveAggregationComplete` - // with `proof=None` and `attestation=None`. Accept it so - // `try_publish_complete` can detect `all_proofs_are_none` and publish. - // Proof aggregation ON: both must be present and verified together. - match (&msg.aggregated_proof, &msg.fold_attestation) { - (None, None) => { - // no-aggregation mode — skip attestation verification - } - (Some(proof), Some(attestation)) => { - let meta = self.params_preset.metadata(); - let committee_n = *circuit_committee_n; - let committee_h = *circuit_committee_h; - let n_moduli = meta.num_moduli; - if committee_n == 0 || committee_h == 0 { - warn!( - party_id = msg.party_id, - "DKG fold attestation verify skipped — circuit committee dims unset" - ); - return Ok(()); - } - if let Err(e) = verify_dkg_fold_attestation( - &self.e3_id, - msg.party_id, - proof, - attestation, - expected_node, - committee_n, - committee_h, - n_moduli, - ) { - warn!( - party_id = msg.party_id, - error = %e, - "DKG fold attestation verification failed — rejecting" - ); - return Ok(()); - } - } - (Some(_), None) => { - warn!( - party_id = msg.party_id, - "DKG fold has proof but missing attestation — rejecting (attribution)" - ); - return Ok(()); - } - (None, Some(_)) => { - warn!( - party_id = msg.party_id, - "DKG fold has attestation but missing proof — rejecting" - ); - return Ok(()); - } - } - } - - info!( - "PublicKeyAggregator: buffered DKG proof from party {} (buffered={})", - msg.party_id, - dkg_node_proofs.len() + 1 - ); - - self.state.try_mutate(&ec, |state| { - let PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - mut dkg_node_proofs, - mut dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - c5_proof_pending, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - last_ec: _, - } = state - else { - return Ok(state); - }; - dkg_node_proofs.insert(msg.party_id, msg.aggregated_proof); - if let Some(attestation) = msg.fold_attestation.clone() { - dkg_fold_attestations.insert(msg.party_id, attestation); - } - Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - c5_proof_pending, - last_ec: Some(ec.clone()), - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - }) - })?; - - self.try_dispatch_nodes_fold_step(&ec) - } - - /// Dispatch the next [`ZkRequest::NodesFoldStep`] if the next slot's proof is buffered - /// and no step is currently in flight. When all H slots are done, calls - /// [`try_dispatch_dkg_aggregation`]. - fn try_dispatch_nodes_fold_step(&mut self, ec: &EventContext) -> Result<()> { - let state = self.state.get(); - let Some(PublicKeyAggregatorState::GeneratingC5Proof { - dkg_node_proofs, - honest_party_ids, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - dkg_aggregation_correlation, - dkg_aggregated_proof, - .. - }) = state.as_ref() - else { - return Ok(()); - }; - - if nodes_fold_step_correlation.is_some() - || dkg_aggregation_correlation.is_some() - || dkg_aggregated_proof.is_some() - { - return Ok(()); - } - - let next_slot = *nodes_fold_completed_slots; - let total_slots = honest_party_ids.len(); - - if next_slot as usize >= total_slots { - return self.try_dispatch_dkg_aggregation(ec); - } - - let Some(&party_id) = honest_party_ids.iter().nth(next_slot as usize) else { - return Ok(()); - }; - - let Some(Some(inner_proof)) = dkg_node_proofs.get(&party_id) else { - return Ok(()); - }; - - let inner_proof = inner_proof.clone(); - let prior_accumulator = nodes_fold_accumulator.clone(); - - let corr = CorrelationId::new(); - self.bus.publish( - ComputeRequest::zk( - ZkRequest::NodesFoldStep(NodesFoldStepRequest { - inner_proof, - prior_accumulator, - slot_index: next_slot, - total_slots, - e3_id: self.e3_id.to_string(), - params_preset: self.params_preset, - committee_size: self.committee_size, - }), - corr, - self.e3_id.clone(), - ), - ec.clone(), - )?; - - info!( - "PublicKeyAggregator: dispatched NodesFoldStep slot={}/{} for E3 {}", - next_slot, total_slots, self.e3_id - ); - - self.state.try_mutate(ec, |state| { - let PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - c5_proof_pending, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation: _, - } = state - else { - return Ok(state); - }; - Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - c5_proof_pending, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation: Some(corr), - }) - })?; - Ok(()) - } - - /// Handle a completed [`ZkResponse::NodesFoldStep`]: advance the accumulator and dispatch - /// the next fold step (or the final DkgAggregation when all H slots are done). - fn handle_nodes_fold_step_response( - &mut self, - correlation_id: CorrelationId, - accumulator_proof: Proof, - ) -> Result<()> { - let state = self.state.get(); - let Some(PublicKeyAggregatorState::GeneratingC5Proof { - nodes_fold_step_correlation, - nodes_fold_completed_slots, - last_ec, - .. - }) = state.as_ref() - else { - return Ok(()); - }; - - if nodes_fold_step_correlation.as_ref() != Some(&correlation_id) { - return Ok(()); - } - - let completed = nodes_fold_completed_slots + 1; - let Some(ec) = last_ec.clone() else { - return Err(anyhow::anyhow!( - "No EventContext for NodesFoldStep response" - )); - }; - - info!( - "PublicKeyAggregator: NodesFoldStep complete (slot {} done) for E3 {}", - completed - 1, - self.e3_id - ); - - self.state.try_mutate_without_context(|state| { - let PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - c5_proof_pending, - last_ec, - nodes_fold_step_correlation: _, - .. - } = state - else { - return Ok(state); - }; - Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - c5_proof_pending, - last_ec, - nodes_fold_accumulator: Some(accumulator_proof), - nodes_fold_completed_slots: completed, - nodes_fold_step_correlation: None, - }) - })?; - - self.try_dispatch_nodes_fold_step(&ec) - } - - /// Dispatch [`ZkRequest::DkgAggregation`] once C5, all honest NodeFold proofs, and the - /// streaming nodes_fold are all ready. - fn try_dispatch_dkg_aggregation(&mut self, ec: &EventContext) -> Result<()> { - let state = self.state.get(); - let Some(PublicKeyAggregatorState::GeneratingC5Proof { - party_nodes, - dkg_node_proofs, - honest_party_ids, - c5_proof_pending, - dkg_aggregation_correlation, - dkg_aggregated_proof, - circuit_committee_n, - circuit_committee_h, - nodes_fold_accumulator, - nodes_fold_completed_slots, - .. - }) = state.as_ref() - else { - return Ok(()); - }; - - let Some(c5_proof) = c5_proof_pending.as_ref() else { - return Ok(()); - }; - - if dkg_aggregation_correlation.is_some() || dkg_aggregated_proof.is_some() { - return Ok(()); - } - - let all_honest_proofs_present = honest_party_ids - .iter() - .all(|id| dkg_node_proofs.contains_key(id)); - if !all_honest_proofs_present { - return Ok(()); - } - - // `proof_aggregation_enabled` is an E3-level flag shared by all nodes, so honest-party - // proofs should be uniformly Some (aggregation on) or uniformly None (aggregation off). - // A mixed bag would silently truncate the dispatched request below; reject it explicitly. - let some_count = honest_party_ids - .iter() - .filter(|id| { - dkg_node_proofs - .get(id) - .map(Option::is_some) - .unwrap_or(false) - }) - .count(); - if some_count != 0 && some_count != honest_party_ids.len() { - error!( - "PublicKeyAggregator: mixed Some/None DKG node proofs across honest parties \ - ({some_count} of {} present); failing E3 {}", - honest_party_ids.len(), - self.e3_id - ); - self.bus.publish( - E3Failed { - e3_id: self.e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DKGInvalidShares, - }, - ec.clone(), - )?; - self.state.try_mutate(ec, |state| { - let PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation: _, - dkg_aggregated_proof, - c5_proof_pending: _, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - } = state - else { - return Ok(state); - }; - - Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation: None, - dkg_aggregated_proof, - c5_proof_pending: None, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - }) - })?; - return Ok(()); - } - - let mut pairs: Vec<_> = dkg_node_proofs - .iter() - .filter(|(pid, _)| honest_party_ids.contains(pid)) - .filter_map(|(pid, p)| p.as_ref().map(|proof| (*pid, proof.clone()))) - .collect(); - pairs.sort_by_key(|(pid, _)| *pid); - let party_ids: Vec = pairs.iter().map(|(pid, _)| *pid).collect(); - let node_fold_proofs: Vec = pairs.into_iter().map(|(_, p)| p).collect(); - info!( - "ORDER-DEBUG dispatch DkgAggregation: honest_party_ids(submission-idx)={:?} \ - dkg_node_proofs_keys(real party_id from DKGRecursiveAggregationComplete)={:?} \ - party_ids_passed_to_circuit={:?}", - honest_party_ids.iter().collect::>(), - { - let mut k: Vec = dkg_node_proofs.keys().copied().collect(); - k.sort(); - k - }, - party_ids - ); - - if node_fold_proofs.is_empty() { - // Proof aggregation disabled. Do NOT call `try_publish_complete` here — it - // is the most common entry into this method, so re-entering it would create - // unbounded mutual recursion (stack overflow in deployed nodes). - info!("PublicKeyAggregator: proof aggregation disabled — skipping DkgAggregation"); - return Ok(()); - } - - // Streaming fold must be complete before dispatching the final aggregation. - let fold_complete = *nodes_fold_completed_slots == honest_party_ids.len() as u32; - if !fold_complete { - return Ok(()); - } - let precomputed_fold = nodes_fold_accumulator.clone(); - - // Build the FULL committee address vector (length N) in ascending party_id order. - // The DKG aggregator circuit's `committee_members: [Field; N_PARTIES]` is the - // committee-hash preimage; passing only the H honest subset would silently - // hash a shorter array and diverge from on-chain `keccak(topNodes)`. - let mut full_committee_party_ids: Vec = party_nodes.keys().copied().collect(); - full_committee_party_ids.sort(); - let committee_addresses = - committee_addresses_in_party_order(&full_committee_party_ids, party_nodes)?; - #[cfg(debug_assertions)] - { - debug_assert_eq!( - committee_addresses.len(), - *circuit_committee_n, - "DkgAggregator committee_addresses must have N entries (full topNodes)" - ); - debug_assert_eq!( - party_ids.len(), - *circuit_committee_h, - "DkgAggregator party_ids must have H entries (honest set)" - ); - } - - let corr = CorrelationId::new(); - self.bus.publish( - ComputeRequest::zk( - ZkRequest::DkgAggregation(DkgAggregationRequest { - node_fold_proofs, - nodes_fold_proof: precomputed_fold, - c5_proof: c5_proof.clone(), - party_ids, - committee_addresses, - params_preset: self.params_preset, - committee_size: self.committee_size, - }), - corr, - self.e3_id.clone(), - ), - ec.clone(), - )?; - - self.state.try_mutate(ec, |state| { - let PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation: _, - dkg_aggregated_proof, - c5_proof_pending, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - } = state - else { - return Ok(state); - }; - Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation: Some(corr), - dkg_aggregated_proof, - c5_proof_pending, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - }) - })?; - Ok(()) - } - - /// Publish `PublicKeyAggregated` when C5 (non-ZK recursive) and, if applicable, the EVM DkgAggregator proof are ready (or aggregation skipped). - fn try_publish_complete(&mut self) -> Result<()> { - if let Some(ec) = self.state.get().and_then(|s| { - if let PublicKeyAggregatorState::GeneratingC5Proof { last_ec, .. } = &s { - last_ec.clone() - } else { - None - } - }) { - self.try_dispatch_dkg_aggregation(&ec)?; - } - - let PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - nodes, - party_nodes, - dkg_fold_attestations, - honest_party_ids, - c5_proof_pending, - dkg_aggregated_proof, - dkg_aggregation_correlation: _, - last_ec, - .. - } = self - .state - .get() - .ok_or_else(|| anyhow::anyhow!("Expected GeneratingC5Proof state"))? - else { - return Ok(()); - }; - - let Some(c5_proof) = c5_proof_pending.as_ref() else { - return Ok(()); - }; - - let all_proofs_are_none = self - .state - .get() - .and_then(|s| { - if let PublicKeyAggregatorState::GeneratingC5Proof { - dkg_node_proofs, - honest_party_ids, - .. - } = &s - { - let all_present = honest_party_ids - .iter() - .all(|id| dkg_node_proofs.contains_key(id)); - Some(all_present && dkg_node_proofs.values().all(|p| p.is_none())) - } else { - None - } - }) - .unwrap_or(false); - - if !all_proofs_are_none && dkg_aggregated_proof.is_none() { - return Ok(()); - } - - let ec = last_ec - .clone() - .ok_or_else(|| anyhow::anyhow!("No EventContext for publish"))?; - - info!( - "Publishing PublicKeyAggregated (dkg_evm_proof={})", - if dkg_aggregated_proof.is_some() { - "present" - } else { - "skipped" - } - ); - - let pk_commitment = extract_pk_commitment(c5_proof)?; - - // Full committee (N entries) — used by on-chain `committee_hash` binding. - let mut full_committee_party_ids: Vec = party_nodes.keys().copied().collect(); - full_committee_party_ids.sort(); - let committee_addresses = - committee_addresses_in_party_order(&full_committee_party_ids, &party_nodes)?; - - // Honest subset (H entries) — used by downstream actors for share-collection gating. - let honest_party_ids_vec: Vec = honest_party_ids.iter().copied().collect(); - let honest_committee_addresses = - committee_addresses_in_party_order(&honest_party_ids_vec, &party_nodes)?; - - let dkg_attestation_bundle = match dkg_aggregated_proof.as_ref() { - Some(_) => { - let bundle = e3_zk_prover::encode_dkg_attestation_bundle( - &honest_party_ids, - &party_nodes, - &dkg_fold_attestations, - )?; - Some(ArcBytes::from_bytes(&bundle)) - } - None => None, - }; - - let event = PublicKeyAggregated { - pubkey: public_key.clone(), - e3_id: self.e3_id.clone(), - nodes: nodes.clone(), - committee_addresses: committee_addresses.clone(), - honest_committee_addresses: honest_committee_addresses.clone(), - pk_commitment, - dkg_aggregator_proof: dkg_aggregated_proof.clone(), - dkg_attestation_bundle, - }; - self.bus.publish(event, ec.clone())?; - - self.state.try_mutate(&ec, |_| { - Ok(PublicKeyAggregatorState::Complete { - public_key, - keyshares: OrderedSet::new(), - nodes, - committee_addresses, - honest_committee_addresses, - }) - })?; - - Ok(()) - } - - fn handle_compute_response(&mut self, msg: TypedEvent) -> Result<()> { - let (msg, _ec) = msg.into_components(); - if msg.e3_id != self.e3_id { - return Ok(()); - } - match msg.response { - ComputeResponseKind::Zk(ZkResponse::NodesFoldStep(resp)) => { - self.handle_nodes_fold_step_response(msg.correlation_id, resp.accumulator_proof)?; - } - ComputeResponseKind::Zk(ZkResponse::DkgAggregation(resp)) => { - let state = self.state.get(); - let Some(PublicKeyAggregatorState::GeneratingC5Proof { last_ec, .. }) = - state.as_ref() - else { - return Ok(()); - }; - let Some(_ec) = last_ec.clone() else { - return Err(anyhow::anyhow!( - "No EventContext for DkgAggregation response" - )); - }; - self.state.try_mutate_without_context(|state| { - let PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - c5_proof_pending, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - } = state - else { - return Ok(state); - }; - if dkg_aggregation_correlation.as_ref() != Some(&msg.correlation_id) { - return Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - c5_proof_pending, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - }); - } - Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation: None, - dkg_aggregated_proof: Some(resp.proof.clone()), - c5_proof_pending, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - }) - })?; - self.try_publish_complete()?; - } - _ => {} - } - Ok(()) - } - - fn handle_compute_request_error(&mut self, msg: TypedEvent) -> Result<()> { - let (msg, ec) = msg.into_components(); - if msg.request().e3_id != self.e3_id { - return Ok(()); - } - - let matched_nodes_fold_step = matches!( - self.state.get(), - Some(PublicKeyAggregatorState::GeneratingC5Proof { - nodes_fold_step_correlation, - .. - }) if nodes_fold_step_correlation.as_ref() == Some(msg.correlation_id()) - ); - - if matched_nodes_fold_step { - error!( - "PublicKeyAggregator: NodesFoldStep failed for E3 {}: {:?}", - self.e3_id, - msg.get_err() - ); - self.bus.publish( - E3Failed { - e3_id: self.e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DKGInvalidShares, - }, - ec.clone(), - )?; - self.state.try_mutate(&ec, |state| { - let PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - c5_proof_pending: _, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation: _, - } = state - else { - return Ok(state); - }; - Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation, - dkg_aggregated_proof, - c5_proof_pending: None, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation: None, - }) - })?; - return Ok(()); - } - - let matched_dkg_aggregation = matches!( - self.state.get(), - Some(PublicKeyAggregatorState::GeneratingC5Proof { - dkg_aggregation_correlation, - .. - }) if dkg_aggregation_correlation.as_ref() == Some(msg.correlation_id()) - ); - - if !matched_dkg_aggregation { - return Ok(()); - } - - error!( - "PublicKeyAggregator: DkgAggregation failed for E3 {}: {:?}", - self.e3_id, - msg.get_err() - ); - - self.bus.publish( - E3Failed { - e3_id: self.e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DKGInvalidShares, - }, - ec.clone(), - )?; - - self.state.try_mutate(&ec, |state| { - let PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation: _, - dkg_aggregated_proof, - c5_proof_pending: _, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - } = state - else { - return Ok(state); - }; - - Ok(PublicKeyAggregatorState::GeneratingC5Proof { - public_key, - keyshare_bytes, - nodes, - party_nodes, - dkg_node_proofs, - dkg_fold_attestations, - honest_party_ids, - dishonest_parties, - circuit_committee_n, - circuit_committee_h, - dkg_aggregation_correlation: None, - dkg_aggregated_proof, - c5_proof_pending: None, - last_ec, - nodes_fold_accumulator, - nodes_fold_completed_slots, - nodes_fold_step_correlation, - }) - })?; - - Ok(()) - } - - pub fn handle_member_expelled( - &mut self, - node: &str, - ec: &EventContext, - ) -> Result<()> { - self.state.try_mutate(ec, |state| { - PublicKeyAggregation::handle_member_expelled(state, node) - }) - } -} - -impl Actor for PublicKeyAggregator { - type Context = Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT); - } -} - -impl Handler for PublicKeyAggregator { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - match msg { - InterfoldEventData::KeyshareCreated(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ShareVerificationComplete(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::PkAggregationProofSigned(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::DKGRecursiveAggregationComplete(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ComputeResponse(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ComputeRequestError(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), - InterfoldEventData::CommitteeMemberExpelled(data) => { - // Only process raw events from chain (party_id not yet resolved). - if data.party_id.is_some() { - return; - } - - let node_addr = data.node.to_string(); - - if data.e3_id != self.e3_id { - error!("Wrong e3_id sent to PublicKeyAggregator for expulsion. This should not happen."); - return; - } - - info!( - "PublicKeyAggregator: committee member expelled: {} for e3_id={}", - node_addr, data.e3_id - ); - trap(EType::PublickeyAggregation, &self.bus.with_ec(&ec), || { - let was_collecting = matches!( - self.state.get(), - Some(PublicKeyAggregatorState::Collecting { .. }) - ); - - self.handle_member_expelled(&node_addr, &ec)?; - - // If we just transitioned to VerifyingC1, dispatch C1 verification - // using the c1_proofs now stored in the VerifyingC1 state (already - // cleaned of the expelled node's entry). - if was_collecting { - if let Some(PublicKeyAggregatorState::VerifyingC1 { - submission_order, - c1_proofs, - .. - }) = self.state.get() - { - self.dispatch_c1_verification( - &submission_order, - &c1_proofs, - ec.clone(), - )?; - } - } - Ok(()) - }); - } - _ => (), - }; - } -} - -impl Handler> for PublicKeyAggregator { - type Result = (); - - fn handle( - &mut self, - event: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - let (event, ec) = event.into_components(); - trap(EType::PublickeyAggregation, &self.bus.with_ec(&ec), || { - let e3_id = event.e3_id.clone(); - let pubkey = event.pubkey.clone(); - let node = event.node.clone(); - let party_id = event.party_id; - let c1_proof = event.signed_pk_generation_proof.clone(); - - if e3_id != self.e3_id { - error!("Wrong e3_id sent to aggregator. This should not happen."); - return Ok(()); - } - - self.add_keyshare(pubkey, node, party_id, c1_proof, &ec)?; - - // If we just transitioned to VerifyingC1, dispatch verification - // using c1_proofs stored in the new state. - if let Some(PublicKeyAggregatorState::VerifyingC1 { - submission_order, - c1_proofs, - .. - }) = self.state.get() - { - self.dispatch_c1_verification(&submission_order, &c1_proofs, ec)?; - } - - Ok(()) - }) - } -} - -impl Handler> for PublicKeyAggregator { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PublickeyAggregation, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_c1_verification_complete(msg), - ) - } -} - -impl Handler> for PublicKeyAggregator { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PublickeyAggregation, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_pk_aggregation_proof_signed(msg), - ) - } -} - -impl Handler> for PublicKeyAggregator { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PublickeyAggregation, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_dkg_recursive_aggregation_complete(msg), - ) - } -} - -impl Handler> for PublicKeyAggregator { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PublickeyAggregation, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_compute_response(msg), - ) - } -} - -impl Handler> for PublicKeyAggregator { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PublickeyAggregation, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_compute_request_error(msg), - ) - } -} - -impl Handler for PublicKeyAggregator { - type Result = (); - fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { - ctx.stop(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use e3_data::{AutoPersist, DataStore, InMemStore, Repository}; - use e3_events::{ - CircuitName, ComputeRequestErrorKind, HistoryCollector, ProofPayload, ProofType, - TakeEvents, Unsequenced, ZkError, - }; - use e3_test_helpers::get_common_setup; - use std::collections::BTreeSet; - - fn test_ctx(data: impl Into) -> EventContext { - EventContext::::from(data.into()).sequence(0) - } - - fn test_state( - initial_state: PublicKeyAggregatorState, - ) -> Persistable { - let repo = Repository::::new(DataStore::from_in_mem( - &InMemStore::new(false).start(), - )); - repo.to_connector().send(Some(initial_state)) - } - - fn dummy_proof(circuit: CircuitName) -> Proof { - Proof::new( - circuit, - ArcBytes::from_bytes(&[1]), - ArcBytes::from_bytes(&[2]), - ) - } - - fn generating_c5_state(correlation_id: CorrelationId) -> PublicKeyAggregatorState { - PublicKeyAggregatorState::GeneratingC5Proof { - public_key: ArcBytes::from_bytes(&[1, 2, 3]), - keyshare_bytes: Vec::new(), - nodes: OrderedSet::new(), - party_nodes: HashMap::new(), - dkg_node_proofs: HashMap::new(), - dkg_fold_attestations: HashMap::new(), - honest_party_ids: BTreeSet::new(), - dishonest_parties: BTreeSet::new(), - circuit_committee_n: 3, - circuit_committee_h: 3, - dkg_aggregation_correlation: Some(correlation_id), - dkg_aggregated_proof: None, - c5_proof_pending: Some(dummy_proof(CircuitName::PkAggregation)), - last_ec: None, - nodes_fold_accumulator: None, - nodes_fold_completed_slots: 0, - nodes_fold_step_correlation: None, - } - } - - async fn build_public_key_aggregator( - initial_state: PublicKeyAggregatorState, - ) -> Result<( - PublicKeyAggregator, - Addr>, - E3id, - )> { - build_public_key_aggregator_with_committee(initial_state, CiphernodesCommitteeSize::Minimum) - .await - } - - async fn build_public_key_aggregator_with_committee( - initial_state: PublicKeyAggregatorState, - committee_size: CiphernodesCommitteeSize, - ) -> Result<( - PublicKeyAggregator, - Addr>, - E3id, - )> { - let (bus, rng, _seed, params, crp, _errors, history) = - get_common_setup(Some(BfvPreset::InsecureThreshold512.into()))?; - let e3_id = E3id::new("42", 1); - let fhe = Arc::new(Fhe::new(params, crp, rng)); - let aggregator = PublicKeyAggregator::new( - PublicKeyAggregatorParams { - fhe, - bus, - e3_id: e3_id.clone(), - params_preset: BfvPreset::InsecureThreshold512, - committee_size, - }, - test_state(initial_state), - ); - - Ok((aggregator, history, e3_id)) - } - - fn c1_proof_with_pk_commitment(e3_id: &E3id, pk_commitment: [u8; 32]) -> SignedProofPayload { - let mut signals = vec![0u8; 96]; - signals[32..64].copy_from_slice(&pk_commitment); - SignedProofPayload { - payload: ProofPayload { - e3_id: e3_id.clone(), - proof_type: ProofType::C1PkGeneration, - proof: Proof::new( - CircuitName::PkGeneration, - ArcBytes::from_bytes(&[1]), - ArcBytes::from_bytes(&signals), - ), - }, - signature: ArcBytes::from_bytes(&[0u8; 65]), - } - } - - fn verifying_c1_non_square_state( - fhe: &Fhe, - e3_id: &E3id, - ) -> Result<(PublicKeyAggregatorState, usize, usize, usize)> { - use fhe::bfv::SecretKey; - use fhe::mbfv::PublicKeyShare; - use fhe_traits::Serialize; - - let committee = CiphernodesCommitteeSize::Micro.values(); - let threshold_n = committee.n; - let threshold_m = committee.threshold; - let circuit_h = committee.h; - assert_ne!( - threshold_n, circuit_h, - "test requires a non-square committee (N != H)" - ); - - let mut submission_order = Vec::with_capacity(threshold_n); - let mut c1_proofs = Vec::with_capacity(threshold_n); - let mut rng = rand::rng(); - - for party_id in 0..threshold_n as u64 { - let node = format!("0x{:040x}", party_id + 1); - if party_id < circuit_h as u64 { - let sk = SecretKey::random(&fhe.params, &mut rng); - let pk_share = PublicKeyShare::new(&sk, fhe.crp.clone(), &mut rng)?; - let ks_bytes = ArcBytes::from_bytes(&pk_share.to_bytes()); - let commitment = e3_zk_helpers::compute_pk_commitment_from_keyshare_bytes( - &ks_bytes, - &fhe.params, - &fhe.crp, - )?; - submission_order.push((party_id, node, ks_bytes)); - c1_proofs.push(Some(c1_proof_with_pk_commitment(e3_id, commitment))); - } else { - submission_order.push((party_id, node, ArcBytes::from_bytes(&[party_id as u8]))); - c1_proofs.push(None); - } - } - - Ok(( - PublicKeyAggregatorState::VerifyingC1 { - submission_order, - threshold_m, - circuit_committee_n: threshold_n, - circuit_committee_h: circuit_h, - c1_proofs, - no_proof_parties: vec![], - }, - threshold_n, - threshold_m, - circuit_h, - )) - } - - async fn next_event( - history: &Addr>, - ) -> Result { - let mut result = history.send(TakeEvents::::new(1)).await?; - assert!(!result.timed_out, "timed out waiting for an event"); - Ok(result.events.pop().expect("expected one event")) - } - - #[actix::test] - async fn dkg_aggregation_compute_error_emits_e3_failed() -> Result<()> { - let correlation_id = CorrelationId::new(); - let (mut aggregator, history, e3_id) = - build_public_key_aggregator(generating_c5_state(correlation_id)).await?; - - let request = ComputeRequest::zk( - ZkRequest::DkgAggregation(DkgAggregationRequest { - node_fold_proofs: vec![dummy_proof(CircuitName::PkAggregation)], - nodes_fold_proof: None, - c5_proof: dummy_proof(CircuitName::PkAggregation), - party_ids: vec![0], - committee_addresses: vec!["0x0000000000000000000000000000000000000001" - .parse() - .expect("test address")], - params_preset: BfvPreset::InsecureThreshold512, - committee_size: CiphernodesCommitteeSize::Minimum, - }), - correlation_id, - e3_id.clone(), - ); - - aggregator.handle_compute_request_error(TypedEvent::new( - ComputeRequestError::new( - ComputeRequestErrorKind::Zk(ZkError::ProofGenerationFailed("boom".to_string())), - request, - ), - test_ctx(E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::None, - reason: FailureReason::None, - }), - ))?; - - let event = next_event(&history).await?; - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == e3_id - && data.failed_at_stage == E3Stage::CommitteeFinalized - && data.reason == FailureReason::DKGInvalidShares - )); - - let Some(PublicKeyAggregatorState::GeneratingC5Proof { - dkg_aggregation_correlation, - c5_proof_pending, - .. - }) = aggregator.state.get() - else { - panic!("expected GeneratingC5Proof state"); - }; - assert!(dkg_aggregation_correlation.is_none()); - assert!(c5_proof_pending.is_none()); - - Ok(()) - } - - #[actix::test] - async fn mixed_dkg_proofs_emit_e3_failed() -> Result<()> { - let correlation_id = CorrelationId::new(); - let mut initial_state = generating_c5_state(correlation_id); - let PublicKeyAggregatorState::GeneratingC5Proof { - ref mut dkg_aggregation_correlation, - ref mut dkg_node_proofs, - ref mut honest_party_ids, - .. - } = initial_state - else { - unreachable!(); - }; - *dkg_aggregation_correlation = None; - honest_party_ids.extend([0, 1]); - dkg_node_proofs.insert(0, Some(dummy_proof(CircuitName::PkAggregation))); - dkg_node_proofs.insert(1, None); - - let (mut aggregator, history, e3_id) = build_public_key_aggregator(initial_state).await?; - let ec = test_ctx(E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::None, - reason: FailureReason::None, - }); - - aggregator.try_dispatch_dkg_aggregation(&ec)?; - - let event = next_event(&history).await?; - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == e3_id - && data.failed_at_stage == E3Stage::CommitteeFinalized - && data.reason == FailureReason::DKGInvalidShares - )); - - let Some(PublicKeyAggregatorState::GeneratingC5Proof { - dkg_aggregation_correlation, - c5_proof_pending, - .. - }) = aggregator.state.get() - else { - panic!("expected GeneratingC5Proof state"); - }; - assert!(dkg_aggregation_correlation.is_none()); - assert!(c5_proof_pending.is_none()); - - Ok(()) - } - - #[actix::test] - async fn honest_dkg_fold_without_attestation_is_not_buffered() -> Result<()> { - let correlation_id = CorrelationId::new(); - let mut initial_state = generating_c5_state(correlation_id); - let PublicKeyAggregatorState::GeneratingC5Proof { - ref mut party_nodes, - ref mut honest_party_ids, - .. - } = initial_state - else { - unreachable!(); - }; - honest_party_ids.insert(2); - party_nodes.insert(2, "0x70997970C51812dc3A010C7d01b50e0d17dc79C8".to_string()); - - let (mut aggregator, _history, e3_id) = build_public_key_aggregator(initial_state).await?; - let ec = test_ctx(DKGRecursiveAggregationComplete { - e3_id: e3_id.clone(), - party_id: 2, - aggregated_proof: Some(dummy_proof(CircuitName::NodeFold)), - fold_attestation: None, - }); - - aggregator.handle_dkg_recursive_aggregation_complete(TypedEvent::new( - DKGRecursiveAggregationComplete { - e3_id: e3_id.clone(), - party_id: 2, - aggregated_proof: Some(dummy_proof(CircuitName::NodeFold)), - fold_attestation: None, - }, - ec, - ))?; - - let Some(PublicKeyAggregatorState::GeneratingC5Proof { - dkg_node_proofs, .. - }) = aggregator.state.get() - else { - panic!("expected GeneratingC5Proof state"); - }; - assert!(!dkg_node_proofs.contains_key(&2)); - - Ok(()) - } - - #[actix::test] - async fn pk_aggregation_proof_pending_carries_canonical_committee_dims() -> Result<()> { - let (bus, rng, _seed, params, crp, _errors, history) = - get_common_setup(Some(BfvPreset::InsecureThreshold512.into()))?; - let e3_id = E3id::new("42", 1); - let fhe = Arc::new(Fhe::new(params, crp, rng)); - let (initial_state, threshold_n, threshold_m, circuit_h) = - verifying_c1_non_square_state(&fhe, &e3_id)?; - - let mut aggregator = PublicKeyAggregator::new( - PublicKeyAggregatorParams { - fhe, - bus, - e3_id: e3_id.clone(), - params_preset: BfvPreset::InsecureThreshold512, - committee_size: CiphernodesCommitteeSize::Micro, - }, - test_state(initial_state), - ); - - let dishonest: BTreeSet = (circuit_h as u64..threshold_n as u64).collect(); - aggregator.handle_c1_verification_complete(TypedEvent::new( - ShareVerificationComplete { - e3_id: e3_id.clone(), - kind: VerificationKind::PkGenerationProofs, - dishonest_parties: dishonest, - }, - test_ctx(ShareVerificationComplete { - e3_id: e3_id.clone(), - kind: VerificationKind::PkGenerationProofs, - dishonest_parties: BTreeSet::new(), - }), - ))?; - - let event = next_event(&history).await?; - assert!(matches!( - event.into_data(), - InterfoldEventData::PkAggregationProofPending(data) - if data.e3_id == e3_id - && data.proof_request.committee_n == threshold_n - && data.proof_request.committee_h == circuit_h - && data.proof_request.committee_threshold == threshold_m - && data.proof_request.keyshare_bytes.len() == circuit_h - )); - - Ok(()) - } -} diff --git a/crates/aggregator/src/actors/threshold_plaintext_aggregator.rs b/crates/aggregator/src/actors/threshold_plaintext_aggregator.rs deleted file mode 100644 index 613dcf456e..0000000000 --- a/crates/aggregator/src/actors/threshold_plaintext_aggregator.rs +++ /dev/null @@ -1,1416 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use std::collections::{BTreeMap, BTreeSet}; -use std::str::FromStr; -use std::time::Duration; - -use crate::domain::threshold_plaintext_aggregation::{ - build_decryption_aggregation_jobs, format_decrypted_plaintext, ThresholdPlaintextAggregation, -}; -use actix::prelude::*; -use actix::SpawnHandle; -use alloy::primitives::Address; -use anyhow::{anyhow, bail, ensure, Result}; -use e3_data::Persistable; -use e3_events::{ - prelude::*, trap, AggregationProofPending, AggregationProofSigned, BusHandle, - CommitteeMemberExpelled, ComputeRequest, ComputeRequestError, ComputeRequestErrorKind, - ComputeResponse, ComputeResponseKind, CorrelationId, DecryptedSharesAggregationProofRequest, - DecryptionAggregationRequest, DecryptionshareCreated, Die, E3Failed, E3Stage, E3id, EType, - EventContext, FailureReason, InterfoldEvent, InterfoldEventData, PlaintextAggregated, Proof, - Sequenced, ShareVerificationComplete, ShareVerificationDispatched, SignedProofPayload, - TypedEvent, VerificationKind, ZkRequest, ZkResponse, -}; -use e3_fhe_params::BfvPreset; -use e3_sortition::{E3CommitteeContainsRequest, E3CommitteeContainsResponse, Sortition}; -use e3_trbfv::{ - calculate_threshold_decryption::CalculateThresholdDecryptionRequest, TrBFVConfig, TrBFVRequest, - TrBFVResponse, -}; -use e3_utils::NotifySync; -use e3_utils::{utility_types::ArcBytes, MAILBOX_LIMIT}; -use e3_zk_helpers::CiphernodesCommitteeSize; -use tracing::{debug, info, trace, warn}; - -/// Env var overriding the decryption-share collection timeout (seconds). -const DECRYPTION_COLLECTION_TIMEOUT_ENV: &str = "E3_DECRYPTION_COLLECTION_TIMEOUT_SECS"; -/// Default wall-clock budget for collecting the honest committee's decryption shares before the -/// round is failed loudly. Without this bound a single absent honest member stalls the decryption -/// round forever (the collector waits for all `H` honest shares with no fallback). -const DEFAULT_DECRYPTION_COLLECTION_TIMEOUT_SECS: u64 = 1800; - -/// Resolve the decryption-share collection timeout, honouring the env override. -fn decryption_collection_timeout() -> Duration { - match std::env::var(DECRYPTION_COLLECTION_TIMEOUT_ENV) - .ok() - .and_then(|v| v.parse::().ok()) - { - Some(secs) => { - info!( - "Decryption-share collection timeout overridden via {}={}s", - DECRYPTION_COLLECTION_TIMEOUT_ENV, secs - ); - Duration::from_secs(secs) - } - None => Duration::from_secs(DEFAULT_DECRYPTION_COLLECTION_TIMEOUT_SECS), - } -} - -/// Internal self-message fired when the decryption-share collection window elapses. -#[derive(Message, Clone, Debug)] -#[rtype(result = "()")] -struct DecryptionCollectionTimeout; - -// Threshold-plaintext aggregation state machine + pure transition logic now live in -// `crate::domain::threshold_plaintext_aggregation`; re-exported here to preserve the public path -// `e3_aggregator::threshold_plaintext_aggregator::*` (and the crate-level glob re-export). -pub use crate::domain::threshold_plaintext_aggregation::{ - Collecting, Complete, Computing, GeneratingC7Proof, ThresholdPlaintextAggregatorState, - VerifyingC6, -}; - -pub struct ThresholdPlaintextAggregator { - bus: BusHandle, - sortition: Addr, - e3_id: E3id, - params_preset: BfvPreset, - committee_size: CiphernodesCommitteeSize, - proof_aggregation_enabled: bool, - state: Persistable, - /// Honest parties' C6 inner proofs (sorted by party id) for [`ZkRequest::DecryptionAggregation`]. - honest_c6_proofs_for_agg: Option)>>, - /// In-flight threshold decryption request. - threshold_decryption_correlation: Option, - /// In-flight decryption aggregation request. - decryption_aggregation_correlation: Option, - /// C7 proofs stored while waiting for decryption aggregation. - c7_proofs_pending: Option>, - /// DecryptionAggregator outputs (set when ZK completes). - decryption_aggregator_proofs: Option>, - /// Last event context, reused for ZK and final publish. - last_ec: Option>, - /// Full registered committee (`topNodes`, length `N`) for decryption-aggregator - /// `committee_hash_*` inputs. Same value as `PublicKeyAggregated.committee_addresses`. - committee_addresses: Vec
, - /// Canonical honest subset from DKG (length `H ≤ N`, from - /// `PublicKeyAggregated.honest_committee_addresses`). Drives share-collection - /// gating (expects one share from each H party) and sender checks after sortition. - honest_committee_addresses: Vec
, - /// Timer handle for the decryption-share collection timeout (cancelled when the actor stops). - timeout_handle: Option, - /// Most recent inbound event context, used as the causal parent for the `E3Failed` event - /// emitted if the collection window elapses while still collecting shares. - timeout_ec: Option>, -} - -pub struct ThresholdPlaintextAggregatorParams { - pub bus: BusHandle, - pub sortition: Addr, - pub e3_id: E3id, - pub params_preset: BfvPreset, - pub committee_size: CiphernodesCommitteeSize, - pub proof_aggregation_enabled: bool, - /// Full committee from `PublicKeyAggregated.committee_addresses` (length `N`). - /// Used for `committee_hash_*` payload binding to on-chain `topNodes`. - pub committee_addresses: Vec
, - /// Honest committee from `PublicKeyAggregated.honest_committee_addresses` - /// (length `H`). Roster for decryption-share collection and sender gating. - pub honest_committee_addresses: Vec
, -} - -fn node_owns_committee_party_slot( - committee: &[Address], - honest_committee: &[Address], - node: &str, - party_id: u64, -) -> bool { - let Some(expected) = usize::try_from(party_id) - .ok() - .and_then(|index| committee.get(index)) - else { - return false; - }; - Address::from_str(node) - .ok() - .is_some_and(|address| &address == expected && honest_committee.contains(&address)) -} - -impl ThresholdPlaintextAggregator { - pub fn new( - params: ThresholdPlaintextAggregatorParams, - state: Persistable, - ) -> Self { - ThresholdPlaintextAggregator { - bus: params.bus, - sortition: params.sortition, - e3_id: params.e3_id, - params_preset: params.params_preset, - committee_size: params.committee_size, - proof_aggregation_enabled: params.proof_aggregation_enabled, - state, - honest_c6_proofs_for_agg: None, - threshold_decryption_correlation: None, - decryption_aggregation_correlation: None, - c7_proofs_pending: None, - decryption_aggregator_proofs: None, - last_ec: None, - committee_addresses: params.committee_addresses, - honest_committee_addresses: params.honest_committee_addresses, - timeout_handle: None, - timeout_ec: None, - } - } - - /// Length of the canonical honest subset (`H`), not on-chain committee size `N`. - /// Share collection waits for one decryption share from each address in - /// `honest_committee_addresses` (sortition membership is checked separately). - fn aggregated_committee_n(&self) -> u64 { - self.honest_committee_addresses.len() as u64 - } - - /// True when `node` owns `party_id` in the full canonical committee and is part of the honest - /// subset selected during DKG. Membership without the slot check permits a real member to - /// relabel a share under another party ID. - fn node_owns_aggregated_pk_party_slot(&self, node: &str, party_id: u64) -> bool { - node_owns_committee_party_slot( - &self.committee_addresses, - &self.honest_committee_addresses, - node, - party_id, - ) - } - - pub fn add_share( - &mut self, - party_id: u64, - share: Vec, - signed_decryption_proofs: Vec, - ec: &EventContext, - ) -> Result<()> { - let required_shares = self.aggregated_committee_n(); - ensure!( - required_shares > 0, - "honest committee addresses must not be empty before collecting decryption shares" - ); - self.state.try_mutate(ec, |state| { - ThresholdPlaintextAggregation::add_share( - state, - party_id, - share.clone(), - signed_decryption_proofs.clone(), - required_shares, - ) - }) - } - - pub fn handle_member_expelled( - &mut self, - party_id: u64, - ec: &EventContext, - ) -> Result<()> { - let required_shares = self.aggregated_committee_n(); - self.state.try_mutate(ec, |state| { - ThresholdPlaintextAggregation::handle_member_expelled(state, party_id, required_shares) - }) - } - - /// Dispatch C6 proof verification through ShareVerificationActor. - pub fn dispatch_c6_verification( - &mut self, - c6_proofs: BTreeMap>, - ec: EventContext, - ) -> Result<()> { - let party_proofs = ThresholdPlaintextAggregation::plan_c6_dispatch(c6_proofs); - - self.bus.publish( - ShareVerificationDispatched { - e3_id: self.e3_id.clone(), - kind: VerificationKind::ThresholdDecryptionProofs, - share_proofs: party_proofs, - decryption_proofs: vec![], - pre_dishonest: BTreeSet::new(), - params_preset: self.params_preset, - committee_size: self.committee_size, - }, - ec, - )?; - Ok(()) - } - - /// Handle ShareVerificationComplete for C6: filter dishonest parties, transition to Computing. - pub fn handle_c6_verification_complete( - &mut self, - msg: TypedEvent, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - - if msg.kind != VerificationKind::ThresholdDecryptionProofs { - return Ok(()); - } - - if msg.e3_id != self.e3_id { - return Ok(()); - } - - let state: VerifyingC6 = self - .state - .get() - .ok_or(anyhow!("Could not get state"))? - .try_into()?; - - let mut dishonest_parties = msg.dishonest_parties.clone(); - if !dishonest_parties.is_empty() { - warn!( - "C6 verification: {} dishonest parties filtered: {:?}", - dishonest_parties.len(), - dishonest_parties - ); - } - - // Filter shares to only honest parties - let mut honest_shares: Vec<(u64, Vec)> = state - .shares - .iter() - .filter(|(id, _)| !dishonest_parties.contains(id)) - .map(|(id, s)| (*id, s.clone())) - .collect(); - - if honest_shares.len() <= state.threshold_m as usize { - warn!( - "Not enough honest shares after C6 verification: {} honest shares, {} required", - honest_shares.len(), - state.threshold_m + 1 - ); - return self.fail_decryption_round(ec); - } - - // Verify each honest party's raw decryption share matches the - // d_commitment attested by their verified C6 proof. Catches the attack - // where a node sends a valid C6 proof for share d_A but broadcasts - // different bytes d_B. - let share_mismatch_parties = - ThresholdPlaintextAggregation::verify_shares_match_c6_commitments( - self.params_preset, - &honest_shares, - &state.c6_proofs, - ); - if !share_mismatch_parties.is_empty() { - warn!( - "C6 share-commitment mismatch for {} parties: {:?} — excluding from aggregation", - share_mismatch_parties.len(), - share_mismatch_parties, - ); - - dishonest_parties.extend(&share_mismatch_parties); - honest_shares.retain(|(id, _)| !share_mismatch_parties.contains(id)); - if honest_shares.len() <= state.threshold_m as usize { - warn!( - "Not enough honest shares after d_commitment check: {} honest, {} required", - honest_shares.len(), - state.threshold_m + 1 - ); - return self.fail_decryption_round(ec); - } - } - - info!( - "C6 verification passed: {} honest parties, transitioning to Computing", - honest_shares.len(), - ); - - // Collect honest C6 inner proofs (from signed payloads) for DecryptionAggregation. - // BTreeMap iteration yields ascending party_id, matching the slot layout - // used by honest_shares above and enforced by decryption_aggregator.nr. - let honest_c6: Vec<(u64, Vec)> = state - .c6_proofs - .iter() - .filter(|(id, _)| !dishonest_parties.contains(id)) - .map(|(id, signed)| { - ( - *id, - signed.iter().map(|s| s.payload.proof.clone()).collect(), - ) - }) - .collect(); - - // Publish ComputeRequest before transitioning state so a publish - // failure leaves us in VerifyingC6 (retryable) rather than - // Computing (no retry path). - // TrBFV scheme size stays N (`threshold_n`); only the share roster is restricted to the - // H canonical honest parties in `PublicKeyAggregated` (see - // `node_owns_aggregated_pk_party_slot`). - let trbfv_config = - TrBFVConfig::new(state.params.clone(), state.threshold_n, state.threshold_m); - - let correlation_id = CorrelationId::new(); - let event = ComputeRequest::trbfv( - TrBFVRequest::CalculateThresholdDecryption(CalculateThresholdDecryptionRequest { - ciphertexts: state.ciphertext_output.clone(), - trbfv_config, - d_share_polys: honest_shares.clone(), - }), - correlation_id, - self.e3_id.clone(), - ); - self.bus.publish(event, ec.clone())?; - - self.honest_c6_proofs_for_agg = Some(honest_c6); - self.threshold_decryption_correlation = Some(correlation_id); - - self.state.try_mutate(&ec, |_| { - Ok(ThresholdPlaintextAggregatorState::Computing(Computing { - shares: honest_shares, - ciphertext_output: state.ciphertext_output, - threshold_m: state.threshold_m, - threshold_n: state.threshold_n, - params: state.params, - })) - })?; - - self.last_ec = Some(ec.clone()); - - Ok(()) - } - - /// Publish AggregationProofPending for C7 proof generation through ProofRequestActor. - pub fn dispatch_c7_proof_request( - &mut self, - shares: Vec<(u64, Vec)>, - plaintext: Vec, - threshold_m: u64, - threshold_n: u64, - ec: EventContext, - ) -> Result<()> { - self.bus.publish( - AggregationProofPending { - e3_id: self.e3_id.clone(), - proof_request: DecryptedSharesAggregationProofRequest { - d_share_polys: shares.clone(), - plaintext: plaintext.clone(), - params_preset: self.params_preset, - threshold_m, - threshold_n, - committee_size: self.committee_size, - }, - plaintext, - shares, - }, - ec, - )?; - Ok(()) - } - - /// Handle AggregationProofSigned: store C7 proofs and wait for C6 fold before publishing. - pub fn handle_aggregation_proof_signed( - &mut self, - msg: TypedEvent, - _ctx: &mut Context, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - - if msg.e3_id != self.e3_id { - return Ok(()); - } - - let state: GeneratingC7Proof = self - .state - .get() - .ok_or(anyhow!("Could not get state"))? - .try_into()?; - - // Extract raw proofs from signed payloads for PlaintextAggregated - let proofs: Vec<_> = msg - .signed_proofs - .iter() - .map(|sp| sp.payload.proof.clone()) - .collect(); - - if proofs.len() != state.plaintext.len() { - warn!( - "C7 proof count mismatch: got {} proofs for {} ciphertext indices", - proofs.len(), - state.plaintext.len() - ); - return self.fail_decryption_round(ec); - } - - info!("C7 proof signed — awaiting DecryptionAggregation..."); - self.c7_proofs_pending = Some(proofs); - self.last_ec = Some(ec.clone()); - self.maybe_start_decryption_aggregation(&ec)?; - self.try_publish_complete() - } - - fn maybe_start_decryption_aggregation(&mut self, ec: &EventContext) -> Result<()> { - if self.c7_proofs_pending.is_none() { - return Ok(()); - } - if self.decryption_aggregator_proofs.is_some() - || self.decryption_aggregation_correlation.is_some() - { - return Ok(()); - } - if !self.proof_aggregation_enabled { - if self.decryption_aggregator_proofs.is_none() { - self.decryption_aggregator_proofs = Some(Vec::new()); - } - return Ok(()); - } - self.dispatch_decryption_aggregation(ec) - } - - fn dispatch_decryption_aggregation(&mut self, ec: &EventContext) -> Result<()> { - if self.committee_addresses.is_empty() { - warn!( - e3_id = %self.e3_id, - "DecryptionAggregation: committee addresses missing at aggregator construction" - ); - return self.fail_decryption_round(ec.clone()); - } - - let Some(c7_proofs) = self.c7_proofs_pending.as_ref() else { - return Ok(()); - }; - if self.decryption_aggregator_proofs.is_some() { - return Ok(()); - } - if self.decryption_aggregation_correlation.is_some() { - return Ok(()); - } - if !self.proof_aggregation_enabled { - self.decryption_aggregator_proofs = Some(Vec::new()); - return Ok(()); - } - let Some(honest_c6) = self.honest_c6_proofs_for_agg.as_ref() else { - warn!( - e3_id = %self.e3_id, - "DecryptionAggregation deferred: honest C6 proofs not retained on aggregator" - ); - return Ok(()); - }; - // With proof aggregation enabled we must have a complete C6 set; otherwise we'd publish - // `decryption_aggregator_proofs = Vec::new()`, which downstream consumers interpret as - // "aggregation disabled". Fail loudly instead so the missing shares are surfaced. - if honest_c6.is_empty() || honest_c6.iter().any(|(_, w)| w.is_empty()) { - warn!( - "DecryptionAggregation: honest C6 inner proofs missing while proof aggregation is enabled" - ); - return self.fail_decryption_round(ec.clone()); - } - let state: GeneratingC7Proof = self - .state - .get() - .ok_or(anyhow!("Could not get state"))? - .try_into()?; - // C6Fold witness width is `T + 1` (same `T` as `threshold_m`). C7 is only proven for the - // first `T + 1` parties after sorting by party id (`handle_decrypted_shares_aggregation_proof` - // truncates); fold slot indices must stay in `0..T+1` and use that same party subset. - let c6_total_slots = state.threshold_m as usize + 1; - if honest_c6.len() < c6_total_slots { - warn!( - "DecryptionAggregation needs at least {} honest C6 parties, have {}", - c6_total_slots, - honest_c6.len() - ); - return self.fail_decryption_round(ec.clone()); - } - let num_ct = c7_proofs.len(); - let Some(jobs) = build_decryption_aggregation_jobs(c7_proofs, honest_c6, c6_total_slots) - else { - return self.fail_decryption_round(ec.clone()); - }; - let corr = CorrelationId::new(); - info!( - e3_id = %self.e3_id, - num_jobs = num_ct, - c6_slots = c6_total_slots, - "DecryptionAggregation: publishing Zk compute request" - ); - self.bus.publish( - ComputeRequest::zk( - ZkRequest::DecryptionAggregation(DecryptionAggregationRequest { - c6_total_slots, - jobs, - committee_addresses: self.committee_addresses.clone(), - params_preset: self.params_preset, - committee_size: self.committee_size, - }), - corr, - self.e3_id.clone(), - ), - ec.clone(), - )?; - self.decryption_aggregation_correlation = Some(corr); - Ok(()) - } - - pub fn handle_compute_response( - &mut self, - msg: TypedEvent, - ctx: &mut Context, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - ensure!( - msg.e3_id == self.e3_id, - "PlaintextAggregator should never receive incorrect e3_id msgs" - ); - - let correlation_id = msg.correlation_id; - match msg.response { - // TrBFV threshold decryption response -> transition to GeneratingC7Proof - ComputeResponseKind::TrBFV(TrBFVResponse::CalculateThresholdDecryption(response)) => { - if self.threshold_decryption_correlation.as_ref() != Some(&correlation_id) { - return Ok(()); - } - self.threshold_decryption_correlation = None; - info!("Received TrBFV threshold decryption response"); - let plaintext = response.plaintext; - - let state: Computing = self - .state - .get() - .ok_or(anyhow!("Could not get state"))? - .try_into()?; - - let shares = state.shares.clone(); - let threshold_m = state.threshold_m; - let threshold_n = state.threshold_n; - - // Publish pending event before transitioning state so a publish - // failure leaves us in Computing (retryable) rather than - // GeneratingC7Proof (no retry path). - self.dispatch_c7_proof_request( - shares.clone(), - plaintext.clone(), - threshold_m, - threshold_n, - ec.clone(), - )?; - - // Transition to GeneratingC7Proof - self.state.try_mutate(&ec, |_| { - Ok(ThresholdPlaintextAggregatorState::GeneratingC7Proof( - GeneratingC7Proof { - threshold_m, - threshold_n, - shares, - plaintext, - }, - )) - })?; - } - - ComputeResponseKind::Zk(ZkResponse::DecryptionAggregation(resp)) => { - if self.decryption_aggregation_correlation.as_ref() == Some(&correlation_id) { - self.decryption_aggregation_correlation = None; - // Worker must return one DecryptionAggregator proof per pending C7 ciphertext. - if let Some(c7_proofs) = self.c7_proofs_pending.as_ref() { - if resp.proofs.len() != c7_proofs.len() { - warn!( - "DecryptionAggregation response proof count {} != expected {}", - resp.proofs.len(), - c7_proofs.len() - ); - return self.fail_decryption_round(ec); - } - } - self.decryption_aggregator_proofs = Some(resp.proofs); - self.try_publish_complete()?; - } - } - - _ => { - // Not a response we handle — ignore - } - } - let _ = ctx; - Ok(()) - } - - fn fail_decryption_round(&mut self, ec: EventContext) -> Result<()> { - self.bus.publish( - E3Failed { - e3_id: self.e3_id.clone(), - failed_at_stage: E3Stage::CiphertextReady, - reason: FailureReason::DecryptionInvalidShares, - }, - ec, - )?; - - self.honest_c6_proofs_for_agg = None; - self.threshold_decryption_correlation = None; - self.decryption_aggregation_correlation = None; - self.c7_proofs_pending = None; - self.decryption_aggregator_proofs = None; - - Ok(()) - } - - fn handle_compute_request_error(&mut self, msg: TypedEvent) -> Result<()> { - let (msg, ec) = msg.into_components(); - if msg.request().e3_id != self.e3_id { - return Ok(()); - } - - let threshold_decryption_failed = - self.threshold_decryption_correlation.as_ref() == Some(msg.correlation_id()); - let decryption_aggregation_failed = - self.decryption_aggregation_correlation.as_ref() == Some(msg.correlation_id()); - - if !threshold_decryption_failed && !decryption_aggregation_failed { - return Ok(()); - } - - // Surface the structured threshold-BFV failure when present so the implicated party and - // failure mode are visible in logs. Slashing/accusation stays driven by the C6 proof - // verification path; this is diagnostics only. - if let ComputeRequestErrorKind::TrBFV(trbfv_err) = msg.get_err() { - let failure = trbfv_err.failure(); - match &failure.threshold { - Some(threshold) => warn!( - e3_id = %self.e3_id, - kind = ?threshold.kind, - party_id = ?threshold.party_id, - "threshold decryption failed with structured error: {}", - threshold.message, - ), - None => warn!( - e3_id = %self.e3_id, - "threshold decryption failed: {}", - failure.message, - ), - } - } - - self.fail_decryption_round(ec) - } - - /// Publish `PlaintextAggregated` when both C7 proofs and decryption aggregation are complete. - fn try_publish_complete(&mut self) -> Result<()> { - let Some(c7_proofs) = self.c7_proofs_pending.clone() else { - return Ok(()); - }; - let dec_ready = self.decryption_aggregator_proofs.is_some() - && self.decryption_aggregation_correlation.is_none(); - if !dec_ready { - return Ok(()); - } - - let state: GeneratingC7Proof = self - .state - .get() - .ok_or_else(|| anyhow!("Expected GeneratingC7Proof state"))? - .try_into()?; - - let ec = self - .last_ec - .clone() - .ok_or_else(|| anyhow!("No EventContext for publish"))?; - - info!("C7 + decryption_aggregator proofs ready — publishing PlaintextAggregated"); - - let decrypted_output = format_decrypted_plaintext(&state.plaintext); - - let decryption_aggregator_proofs = self - .decryption_aggregator_proofs - .clone() - .unwrap_or_default(); - // Keep c7_proofs for invariant check; they are subsumed by the decryption_aggregator proof. - let _ = c7_proofs; - let event = PlaintextAggregated { - decrypted_output, - e3_id: self.e3_id.clone(), - decryption_aggregator_proofs, - }; - - info!("Dispatching plaintext event {:?}", event); - self.bus.publish(event, ec.clone())?; - - self.state.try_mutate(&ec, |_| { - Ok(ThresholdPlaintextAggregatorState::Complete(Complete { - decrypted: state.plaintext, - shares: state.shares, - })) - })?; - - Ok(()) - } -} - -impl Actor for ThresholdPlaintextAggregator { - type Context = Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT); - // Bound the decryption-share collection phase so a missing honest member cannot stall the - // round indefinitely. On expiry the round is failed loudly (see the timeout handler). - let timeout = decryption_collection_timeout(); - info!( - e3_id = %self.e3_id, - ?timeout, - "ThresholdPlaintextAggregator started; scheduling decryption-share collection timeout" - ); - self.timeout_handle = Some(ctx.notify_later(DecryptionCollectionTimeout, timeout)); - } -} - -impl Handler for ThresholdPlaintextAggregator { - type Result = (); - fn handle(&mut self, _: DecryptionCollectionTimeout, ctx: &mut Self::Context) -> Self::Result { - self.timeout_handle = None; - - // Only fail while still collecting shares; once we have transitioned past `Collecting` - // (VerifyingC6/Computing/…) the round is progressing and the timer is a no-op. - let Some(ThresholdPlaintextAggregatorState::Collecting(collecting)) = self.state.get() - else { - debug!( - e3_id = %self.e3_id, - "Decryption-share collection timeout fired but round already progressed past collection; ignoring" - ); - return; - }; - - let collected = collecting.shares.len(); - let required = self.aggregated_committee_n(); - warn!( - e3_id = %self.e3_id, - collected, - required, - "Decryption-share collection timed out with {collected}/{required} honest shares; failing E3 round (DecryptionTimeout)" - ); - - let Some(ec) = self.timeout_ec.clone() else { - warn!( - e3_id = %self.e3_id, - "No event context captured for decryption timeout; cannot emit E3Failed. Stopping aggregator." - ); - ctx.stop(); - return; - }; - - if let Err(e) = self.bus.publish( - E3Failed { - e3_id: self.e3_id.clone(), - failed_at_stage: E3Stage::CiphertextReady, - reason: FailureReason::DecryptionTimeout, - }, - ec, - ) { - warn!( - e3_id = %self.e3_id, - error = %e, - "Failed to publish E3Failed on decryption-share collection timeout" - ); - } - - ctx.stop(); - } -} - -impl Handler for ThresholdPlaintextAggregator { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - match msg { - InterfoldEventData::DecryptionshareCreated(data) => { - ctx.notify(TypedEvent::new(data, ec)) - } - InterfoldEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), - InterfoldEventData::ComputeResponse(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ComputeRequestError(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::CommitteeMemberExpelled(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ShareVerificationComplete(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::AggregationProofSigned(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - _ => (), - } - } -} - -impl Handler> for ThresholdPlaintextAggregator { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PublickeyAggregation, - &self.bus.with_ec(msg.get_ctx()), - || { - let Some(ThresholdPlaintextAggregatorState::Collecting(Collecting { .. })) = - self.state.get() - else { - debug!(state=?self.state, "Aggregator has been closed for collecting so ignoring this event."); - return Ok(()); - }; - let node = msg.node.clone(); - let e3_id = msg.e3_id.clone(); - let request = E3CommitteeContainsRequest::new(e3_id, node, msg, ctx.address()); - self.sortition.try_send(request)?; - Ok(()) - }, - ) - } -} - -impl Handler>> - for ThresholdPlaintextAggregator -{ - type Result = (); - fn handle( - &mut self, - msg: E3CommitteeContainsResponse>, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PublickeyAggregation, - &self.bus.with_ec(msg.get_ctx()), - || { - let e3_id = &msg.e3_id; - if *e3_id != self.e3_id { - bail!("Wrong e3_id sent to aggregator. This should not happen.") - }; - - if !msg.is_found_in_committee() { - trace!("Node {} not found in finalized committee", &msg.node); - return Ok(()); - }; - if !self.node_owns_aggregated_pk_party_slot(&msg.node, msg.party_id) { - trace!( - "Node {} does not own honest party slot {} — ignoring decryption share", - &msg.node, - msg.party_id - ); - return Ok(()); - } - - // Trust the party_id from the event - it's based on CommitteeFinalized order - // which is the authoritative source of truth for party IDs - let ( - DecryptionshareCreated { - party_id, - decryption_share, - signed_decryption_proofs, - .. - }, - ec, - ) = msg.into_inner().into_components(); - - // Capture the latest context so a subsequent collection timeout can emit - // `E3Failed` with a sensible causal parent. - self.timeout_ec = Some(ec.clone()); - self.add_share(party_id, decryption_share, signed_decryption_proofs, &ec)?; - - // If we transitioned to VerifyingC6, dispatch C6 verification - // using the proofs persisted in state - if let Some(ThresholdPlaintextAggregatorState::VerifyingC6(ref state)) = - self.state.get() - { - self.dispatch_c6_verification(state.c6_proofs.clone(), ec)?; - } - - Ok(()) - }, - ) - } -} - -impl Handler> for ThresholdPlaintextAggregator { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PlaintextAggregation, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_compute_response(msg, ctx), - ) - } -} - -impl Handler> for ThresholdPlaintextAggregator { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PlaintextAggregation, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_compute_request_error(msg), - ) - } -} - -impl Handler> for ThresholdPlaintextAggregator { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PlaintextAggregation, - &self.bus.with_ec(msg.get_ctx()), - || { - let (msg, ec) = msg.into_components(); - let Some(party_id) = msg.party_id else { - return Ok(()); - }; - - self.handle_member_expelled(party_id, &ec)?; - - if let Some(ThresholdPlaintextAggregatorState::VerifyingC6(ref state)) = - self.state.get() - { - self.dispatch_c6_verification(state.c6_proofs.clone(), ec)?; - } - - Ok(()) - }, - ) - } -} - -impl Handler> for ThresholdPlaintextAggregator { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PlaintextAggregation, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_c6_verification_complete(msg), - ) - } -} - -impl Handler> for ThresholdPlaintextAggregator { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::PlaintextAggregation, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_aggregation_proof_signed(msg, ctx), - ) - } -} - -impl Handler for ThresholdPlaintextAggregator { - type Result = (); - fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { - ctx.stop() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use e3_data::{AutoPersist, DataStore, InMemStore, PersistableData, Repository}; - use e3_events::{ - CircuitName, Committee, ComputeRequestErrorKind, HistoryCollector, Seed, TakeEvents, - Unsequenced, ZkError, - }; - use e3_fhe_params::{encode_bfv_params, BfvParamSet, DEFAULT_BFV_PRESET}; - use e3_sortition::{ - CiphernodeSelector, CiphernodeSelectorState, NodeStateStore, SortitionBackend, - SortitionParams, - }; - use e3_test_helpers::get_common_setup; - use std::collections::{BTreeMap, BTreeSet, HashMap}; - - fn test_ctx(data: impl Into) -> EventContext { - EventContext::::from(data.into()).sequence(0) - } - - fn test_persistable(value: T) -> Persistable { - let repo = Repository::::new(DataStore::from_in_mem(&InMemStore::new(false).start())); - repo.to_connector().send(Some(value)) - } - - fn test_params() -> ArcBytes { - ArcBytes::from_bytes(&encode_bfv_params( - &BfvParamSet::from(DEFAULT_BFV_PRESET).build_arc(), - )) - } - - fn dummy_proof(circuit: CircuitName) -> Proof { - Proof::new( - circuit, - ArcBytes::from_bytes(&[1]), - ArcBytes::from_bytes(&[2]), - ) - } - - #[test] - fn decryption_sender_must_own_an_honest_party_slot() { - let first = Address::repeat_byte(0x11); - let second = Address::repeat_byte(0x22); - let committee = [first, second]; - let honest = [first]; - - assert!(node_owns_committee_party_slot( - &committee, - &honest, - &first.to_string(), - 0 - )); - assert!(!node_owns_committee_party_slot( - &committee, - &honest, - &first.to_string(), - 1 - )); - assert!(!node_owns_committee_party_slot( - &committee, - &honest, - &second.to_string(), - 1 - )); - assert!(!node_owns_committee_party_slot( - &committee, - &honest, - &first.to_string(), - 2 - )); - } - - fn computing_state() -> ThresholdPlaintextAggregatorState { - ThresholdPlaintextAggregatorState::Computing(Computing { - threshold_m: 1, - threshold_n: 2, - shares: vec![(0, vec![ArcBytes::from_bytes(&[7])])], - ciphertext_output: vec![ArcBytes::from_bytes(&[8])], - params: test_params(), - }) - } - - fn verifying_c6_state() -> ThresholdPlaintextAggregatorState { - ThresholdPlaintextAggregatorState::VerifyingC6(VerifyingC6 { - threshold_m: 1, - threshold_n: 2, - shares: BTreeMap::from([ - (0, vec![ArcBytes::from_bytes(&[7])]), - (1, vec![ArcBytes::from_bytes(&[8])]), - ]), - c6_proofs: BTreeMap::new(), - ciphertext_output: vec![ArcBytes::from_bytes(&[9])], - params: test_params(), - }) - } - - fn generating_c7_state() -> ThresholdPlaintextAggregatorState { - ThresholdPlaintextAggregatorState::GeneratingC7Proof(GeneratingC7Proof { - threshold_m: 1, - threshold_n: 2, - shares: vec![(0, vec![ArcBytes::from_bytes(&[7])])], - plaintext: vec![ArcBytes::from_bytes(&[9])], - }) - } - - fn collecting_state() -> ThresholdPlaintextAggregatorState { - ThresholdPlaintextAggregatorState::Collecting(Collecting { - threshold_m: 1, - threshold_n: 2, - shares: BTreeMap::new(), - c6_proofs: BTreeMap::new(), - seed: Seed([0u8; 32]), - ciphertext_output: vec![ArcBytes::from_bytes(&[9])], - params: test_params(), - }) - } - - fn start_sortition(bus: &BusHandle) -> Addr { - let selector = CiphernodeSelector::new( - bus, - test_persistable(CiphernodeSelectorState::default()), - "node-1", - ) - .start(); - - Sortition::new(SortitionParams { - bus: bus.clone(), - backends: test_persistable(HashMap::::new()), - node_state: test_persistable(HashMap::::new()), - finalized_committees: test_persistable(HashMap::::new()), - ciphernode_selector: selector, - address: "node-1".to_string(), - }) - .start() - } - - fn test_committee_address() -> Address { - "0x0000000000000000000000000000000000000001" - .parse() - .expect("test address") - } - - async fn build_plaintext_aggregator( - initial_state: ThresholdPlaintextAggregatorState, - proof_aggregation_enabled: bool, - ) -> Result<( - ThresholdPlaintextAggregator, - Addr>, - E3id, - )> { - let (bus, _rng, _seed, _params, _crp, _errors, history) = - get_common_setup(Some(BfvPreset::InsecureThreshold512.into()))?; - let e3_id = E3id::new("42", 1); - let aggregator = ThresholdPlaintextAggregator::new( - ThresholdPlaintextAggregatorParams { - bus: bus.clone(), - sortition: start_sortition(&bus), - e3_id: e3_id.clone(), - params_preset: BfvPreset::InsecureThreshold512, - committee_size: CiphernodesCommitteeSize::Minimum, - proof_aggregation_enabled, - committee_addresses: vec![test_committee_address()], - honest_committee_addresses: vec![test_committee_address()], - }, - test_persistable(initial_state), - ); - - Ok((aggregator, history, e3_id)) - } - - async fn next_event( - history: &Addr>, - ) -> Result { - let mut result = history.send(TakeEvents::::new(1)).await?; - assert!(!result.timed_out, "timed out waiting for an event"); - Ok(result.events.pop().expect("expected one event")) - } - - #[actix::test] - async fn decryption_collection_timeout_fails_round_while_collecting() -> Result<()> { - let (mut aggregator, history, e3_id) = - build_plaintext_aggregator(collecting_state(), true).await?; - // A captured context is required so the timeout can emit E3Failed with a causal parent. - aggregator.timeout_ec = Some(test_ctx(E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CiphertextReady, - reason: FailureReason::None, - })); - let addr = aggregator.start(); - - addr.send(DecryptionCollectionTimeout).await?; - - let event = next_event(&history).await?; - assert!( - matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.reason == FailureReason::DecryptionTimeout - ), - "expected E3Failed with DecryptionTimeout when collection window elapses" - ); - Ok(()) - } - - #[actix::test] - async fn threshold_decryption_compute_error_emits_e3_failed() -> Result<()> { - let correlation_id = CorrelationId::new(); - let (mut aggregator, history, e3_id) = - build_plaintext_aggregator(computing_state(), true).await?; - aggregator.threshold_decryption_correlation = Some(correlation_id); - - let request = ComputeRequest::trbfv( - TrBFVRequest::CalculateThresholdDecryption(CalculateThresholdDecryptionRequest { - ciphertexts: vec![ArcBytes::from_bytes(&[8])], - trbfv_config: TrBFVConfig::new(test_params(), 2, 1), - d_share_polys: vec![(0, vec![ArcBytes::from_bytes(&[7])])], - }), - correlation_id, - e3_id.clone(), - ); - - aggregator.handle_compute_request_error(TypedEvent::new( - ComputeRequestError::new( - ComputeRequestErrorKind::TrBFV(e3_trbfv::TrBFVError::CalculateThresholdDecryption( - "boom".into(), - )), - request, - ), - test_ctx(E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::None, - reason: FailureReason::None, - }), - ))?; - - let event = next_event(&history).await?; - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == e3_id - && data.failed_at_stage == E3Stage::CiphertextReady - && data.reason == FailureReason::DecryptionInvalidShares - )); - assert!(aggregator.threshold_decryption_correlation.is_none()); - - Ok(()) - } - - #[actix::test] - async fn insufficient_honest_c6_shares_emit_e3_failed() -> Result<()> { - let (mut aggregator, history, e3_id) = - build_plaintext_aggregator(verifying_c6_state(), true).await?; - - aggregator.handle_c6_verification_complete(TypedEvent::new( - ShareVerificationComplete { - e3_id: e3_id.clone(), - kind: VerificationKind::ThresholdDecryptionProofs, - dishonest_parties: BTreeSet::from([1]), - }, - test_ctx(E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::None, - reason: FailureReason::None, - }), - ))?; - - let event = next_event(&history).await?; - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == e3_id - && data.failed_at_stage == E3Stage::CiphertextReady - && data.reason == FailureReason::DecryptionInvalidShares - )); - - Ok(()) - } - - #[actix::test] - async fn decryption_aggregation_compute_error_emits_e3_failed() -> Result<()> { - let correlation_id = CorrelationId::new(); - let (mut aggregator, history, e3_id) = - build_plaintext_aggregator(generating_c7_state(), true).await?; - aggregator.c7_proofs_pending = Some(vec![dummy_proof(CircuitName::PkAggregation)]); - aggregator.honest_c6_proofs_for_agg = Some(vec![( - 0, - vec![dummy_proof(CircuitName::ThresholdShareDecryption)], - )]); - aggregator.decryption_aggregation_correlation = Some(correlation_id); - aggregator.last_ec = Some(test_ctx(E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::None, - reason: FailureReason::None, - })); - - let request = ComputeRequest::zk( - ZkRequest::DecryptionAggregation(DecryptionAggregationRequest { - c6_total_slots: 1, - jobs: Vec::new(), - committee_addresses: vec![test_committee_address()], - params_preset: BfvPreset::InsecureThreshold512, - committee_size: CiphernodesCommitteeSize::Minimum, - }), - correlation_id, - e3_id.clone(), - ); - - aggregator.handle_compute_request_error(TypedEvent::new( - ComputeRequestError::new( - ComputeRequestErrorKind::Zk(ZkError::ProofGenerationFailed("boom".to_string())), - request, - ), - test_ctx(E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::None, - reason: FailureReason::None, - }), - ))?; - - let event = next_event(&history).await?; - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == e3_id - && data.failed_at_stage == E3Stage::CiphertextReady - && data.reason == FailureReason::DecryptionInvalidShares - )); - assert!(aggregator.decryption_aggregation_correlation.is_none()); - assert!(aggregator.c7_proofs_pending.is_none()); - - Ok(()) - } - - #[actix::test] - async fn missing_c6_inner_proofs_emit_e3_failed() -> Result<()> { - let (mut aggregator, history, e3_id) = - build_plaintext_aggregator(generating_c7_state(), true).await?; - aggregator.c7_proofs_pending = Some(vec![dummy_proof(CircuitName::PkAggregation)]); - aggregator.honest_c6_proofs_for_agg = Some(vec![ - (0, vec![]), - (1, vec![dummy_proof(CircuitName::ThresholdShareDecryption)]), - ]); - - let ec = test_ctx(E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::None, - reason: FailureReason::None, - }); - aggregator.dispatch_decryption_aggregation(&ec)?; - - let event = next_event(&history).await?; - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == e3_id - && data.failed_at_stage == E3Stage::CiphertextReady - && data.reason == FailureReason::DecryptionInvalidShares - )); - assert!(aggregator.honest_c6_proofs_for_agg.is_none()); - assert!(aggregator.decryption_aggregation_correlation.is_none()); - assert!(aggregator.c7_proofs_pending.is_none()); - assert!(aggregator.decryption_aggregator_proofs.is_none()); - - Ok(()) - } - - #[actix::test] - async fn proof_aggregation_disabled_marks_decryption_aggregator_ready() -> Result<()> { - let (mut aggregator, _history, _e3_id) = - build_plaintext_aggregator(generating_c7_state(), false).await?; - aggregator.c7_proofs_pending = Some(vec![dummy_proof(CircuitName::PkAggregation)]); - let ec = test_ctx(E3Failed { - e3_id: aggregator.e3_id.clone(), - failed_at_stage: E3Stage::None, - reason: FailureReason::None, - }); - - aggregator.dispatch_decryption_aggregation(&ec)?; - assert!(aggregator - .decryption_aggregator_proofs - .as_ref() - .is_some_and(|p| p.is_empty())); - assert!(aggregator.decryption_aggregation_correlation.is_none()); - - Ok(()) - } -} diff --git a/crates/aggregator/src/domain/committee.rs b/crates/aggregator/src/committee.rs similarity index 100% rename from crates/aggregator/src/domain/committee.rs rename to crates/aggregator/src/committee.rs diff --git a/crates/aggregator/src/actors/committee_finalizer.rs b/crates/aggregator/src/committee_finalization/actor.rs similarity index 53% rename from crates/aggregator/src/actors/committee_finalizer.rs rename to crates/aggregator/src/committee_finalization/actor.rs index a42820b7a3..c1295d8f90 100644 --- a/crates/aggregator/src/actors/committee_finalizer.rs +++ b/crates/aggregator/src/committee_finalization/actor.rs @@ -16,6 +16,9 @@ use std::collections::HashMap; use std::time::Duration; use tracing::{error, info}; +#[path = "handlers.rs"] +mod handlers; + const FINALIZATION_BUFFER_SECONDS: u64 = 1; const FINALIZE_INTERVAL_SECONDS: u64 = 5; @@ -166,142 +169,3 @@ impl Actor for CommitteeFinalizer { ctx.set_mailbox_capacity(MAILBOX_LIMIT); } } - -impl Handler for CommitteeFinalizer { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - match msg { - InterfoldEventData::CommitteeRequested(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::EffectsEnabled(data) => self.notify_sync(ctx, data), - InterfoldEventData::TicketGenerated(data) => self.notify_sync(ctx, data), - InterfoldEventData::Shutdown(data) => self.notify_sync(ctx, data), - InterfoldEventData::E3Failed(data) => self.notify_sync(ctx, TypedEvent::new(data, ec)), - InterfoldEventData::E3RequestComplete(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::E3StageChanged(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - _ => (), - } - } -} - -impl Handler> for CommitteeFinalizer { - type Result = (); - - // TODO: Remove all async from this function. Remove reliance on e3_evm package. Add unit test. - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - let e3_id = msg.e3_id.to_string(); - self.pending_requests.insert( - e3_id.clone(), - PendingCommitteeRequest { - e3_id: msg.e3_id.clone(), - committee_deadline: msg.committee_deadline, - ec: msg.get_ctx().clone(), - }, - ); - self.schedule_if_ready(&e3_id, ctx); - } -} - -impl Handler for CommitteeFinalizer { - type Result = (); - - fn handle(&mut self, msg: TicketGenerated, ctx: &mut Self::Context) -> Self::Result { - let Some(party_index) = msg.party_index else { - return; - }; - - let e3_id = msg.e3_id.to_string(); - self.party_indexes.insert(e3_id.clone(), party_index); - self.schedule_if_ready(&e3_id, ctx); - } -} - -impl Handler for CommitteeFinalizer { - type Result = (); - - fn handle(&mut self, _msg: EffectsEnabled, ctx: &mut Self::Context) -> Self::Result { - self.effects_enabled = true; - let e3_ids: Vec = self.pending_requests.keys().cloned().collect(); - for e3_id in e3_ids { - self.schedule_if_ready(&e3_id, ctx); - } - } -} - -impl Handler for CommitteeFinalizer { - type Result = (); - fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { - info!("Killing CommitteeFinalizer"); - // Cancel all pending finalization tasks - for (_, handle) in self.pending_committees.drain() { - ctx.cancel_future(handle); - } - ctx.stop(); - } -} - -impl Handler> for CommitteeFinalizer { - type Result = (); - fn handle(&mut self, msg: TypedEvent, ctx: &mut Self::Context) -> Self::Result { - let e3_id_str = msg.e3_id.to_string(); - if let Some(handle) = self.pending_committees.remove(&e3_id_str) { - info!( - e3_id = %msg.e3_id, - reason = ?msg.reason, - "E3 failed — cancelling pending committee finalization timer" - ); - ctx.cancel_future(handle); - } - self.pending_requests.remove(&e3_id_str); - self.party_indexes.remove(&e3_id_str); - } -} - -impl Handler> for CommitteeFinalizer { - type Result = (); - fn handle(&mut self, msg: TypedEvent, ctx: &mut Self::Context) -> Self::Result { - match &msg.new_stage { - E3Stage::Complete | E3Stage::Failed => { - let e3_id_str = msg.e3_id.to_string(); - if let Some(handle) = self.pending_committees.remove(&e3_id_str) { - info!( - e3_id = %msg.e3_id, - stage = ?msg.new_stage, - "E3 reached terminal stage — cancelling pending committee finalization timer" - ); - ctx.cancel_future(handle); - } - self.pending_requests.remove(&e3_id_str); - self.party_indexes.remove(&e3_id_str); - } - _ => {} - } - } -} - -impl Handler> for CommitteeFinalizer { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - let e3_id_str = msg.e3_id.to_string(); - if let Some(handle) = self.pending_committees.remove(&e3_id_str) { - ctx.cancel_future(handle); - } - self.pending_requests.remove(&e3_id_str); - self.party_indexes.remove(&e3_id_str); - } -} diff --git a/crates/aggregator/src/committee_finalization/handlers.rs b/crates/aggregator/src/committee_finalization/handlers.rs new file mode 100644 index 0000000000..d225403eae --- /dev/null +++ b/crates/aggregator/src/committee_finalization/handlers.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Committee deadline, cleanup, and lifecycle handlers. + +use super::*; + +impl Handler for CommitteeFinalizer { + type Result = (); + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + match msg { + InterfoldEventData::CommitteeRequested(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::EffectsEnabled(data) => self.notify_sync(ctx, data), + InterfoldEventData::TicketGenerated(data) => self.notify_sync(ctx, data), + InterfoldEventData::Shutdown(data) => self.notify_sync(ctx, data), + InterfoldEventData::E3Failed(data) => self.notify_sync(ctx, TypedEvent::new(data, ec)), + InterfoldEventData::E3RequestComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::E3StageChanged(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + _ => (), + } + } +} + +impl Handler> for CommitteeFinalizer { + type Result = (); + + // TODO: Remove all async from this function. Remove reliance on e3_evm package. Add unit test. + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let e3_id = msg.e3_id.to_string(); + self.pending_requests.insert( + e3_id.clone(), + PendingCommitteeRequest { + e3_id: msg.e3_id.clone(), + committee_deadline: msg.committee_deadline, + ec: msg.get_ctx().clone(), + }, + ); + self.schedule_if_ready(&e3_id, ctx); + } +} + +impl Handler for CommitteeFinalizer { + type Result = (); + + fn handle(&mut self, msg: TicketGenerated, ctx: &mut Self::Context) -> Self::Result { + let Some(party_index) = msg.party_index else { + return; + }; + + let e3_id = msg.e3_id.to_string(); + self.party_indexes.insert(e3_id.clone(), party_index); + self.schedule_if_ready(&e3_id, ctx); + } +} + +impl Handler for CommitteeFinalizer { + type Result = (); + + fn handle(&mut self, _msg: EffectsEnabled, ctx: &mut Self::Context) -> Self::Result { + self.effects_enabled = true; + let e3_ids: Vec = self.pending_requests.keys().cloned().collect(); + for e3_id in e3_ids { + self.schedule_if_ready(&e3_id, ctx); + } + } +} + +impl Handler for CommitteeFinalizer { + type Result = (); + fn handle(&mut self, _msg: Shutdown, ctx: &mut Self::Context) -> Self::Result { + info!("Killing CommitteeFinalizer"); + // Cancel all pending finalization tasks + for (_, handle) in self.pending_committees.drain() { + ctx.cancel_future(handle); + } + ctx.stop(); + } +} + +impl Handler> for CommitteeFinalizer { + type Result = (); + fn handle(&mut self, msg: TypedEvent, ctx: &mut Self::Context) -> Self::Result { + let e3_id_str = msg.e3_id.to_string(); + if let Some(handle) = self.pending_committees.remove(&e3_id_str) { + info!( + e3_id = %msg.e3_id, + reason = ?msg.reason, + "E3 failed — cancelling pending committee finalization timer" + ); + ctx.cancel_future(handle); + } + self.pending_requests.remove(&e3_id_str); + self.party_indexes.remove(&e3_id_str); + } +} + +impl Handler> for CommitteeFinalizer { + type Result = (); + fn handle(&mut self, msg: TypedEvent, ctx: &mut Self::Context) -> Self::Result { + match &msg.new_stage { + E3Stage::Complete | E3Stage::Failed => { + let e3_id_str = msg.e3_id.to_string(); + if let Some(handle) = self.pending_committees.remove(&e3_id_str) { + info!( + e3_id = %msg.e3_id, + stage = ?msg.new_stage, + "E3 reached terminal stage — cancelling pending committee finalization timer" + ); + ctx.cancel_future(handle); + } + self.pending_requests.remove(&e3_id_str); + self.party_indexes.remove(&e3_id_str); + } + _ => {} + } + } +} + +impl Handler> for CommitteeFinalizer { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let e3_id_str = msg.e3_id.to_string(); + if let Some(handle) = self.pending_committees.remove(&e3_id_str) { + ctx.cancel_future(handle); + } + self.pending_requests.remove(&e3_id_str); + self.party_indexes.remove(&e3_id_str); + } +} diff --git a/crates/aggregator/src/domain/committee_hash.rs b/crates/aggregator/src/committee_hash.rs similarity index 100% rename from crates/aggregator/src/domain/committee_hash.rs rename to crates/aggregator/src/committee_hash.rs diff --git a/crates/aggregator/src/domain.rs b/crates/aggregator/src/domain.rs new file mode 100644 index 0000000000..0ba8d70c7b --- /dev/null +++ b/crates/aggregator/src/domain.rs @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Compatibility view of pure aggregation modules stored by capability. + +#[path = "committee.rs"] +pub mod committee; +#[path = "committee_hash.rs"] +pub mod committee_hash; +#[path = "failover.rs"] +pub mod failover; diff --git a/crates/aggregator/src/domain/mod.rs b/crates/aggregator/src/domain/mod.rs deleted file mode 100644 index 559a569809..0000000000 --- a/crates/aggregator/src/domain/mod.rs +++ /dev/null @@ -1,15 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Pure, synchronous domain logic for aggregation. These modules contain no -//! actix, persistence, or event-bus dependencies and are unit-tested in -//! isolation. The actors in [`crate::actors`] drive them and perform all I/O. - -pub mod committee; -pub mod committee_hash; -pub mod failover; -pub mod publickey_aggregation; -pub mod threshold_plaintext_aggregation; diff --git a/crates/aggregator/src/domain/publickey_aggregation.rs b/crates/aggregator/src/domain/publickey_aggregation.rs deleted file mode 100644 index 18f886a2b7..0000000000 --- a/crates/aggregator/src/domain/publickey_aggregation.rs +++ /dev/null @@ -1,750 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Plain, synchronous domain logic for public-key aggregation. -//! -//! This module holds the [`PublicKeyAggregatorState`] state machine plus the pure -//! transition/decision functions used by the `PublicKeyAggregator` actor. Nothing here -//! touches actix, `Persistable`, or the event bus: the actor feeds inputs in, gets a -//! next-state or a [decision](C1Dispatch)/[`HonestSelection`] back, and performs the -//! persistence/publish/dispatch side effects itself. - -use alloy::primitives::Address; -use anyhow::{anyhow, ensure, Context as _, Result}; -use e3_events::{ - CircuitName, E3id, OrderedSet, PartyProofsToVerify, Proof, Seed, SignedDkgFoldAttestation, - SignedProofPayload, -}; -use e3_fhe::Fhe; -use e3_utils::ArcBytes; -use e3_zk_helpers::cap_honest_party_ids; -use e3_zk_helpers::CiphernodesCommitteeSize; -use e3_zk_prover::extract_node_fold_agg_commits; -use std::collections::{BTreeSet, HashMap}; -use tracing::{error, info, warn}; - -/// Circuit honest-party count `H` for the committee `(threshold_m, threshold_n)`. -pub(crate) fn committee_h_for(threshold_m: usize, threshold_n: usize) -> Result { - Ok( - CiphernodesCommitteeSize::from_threshold(threshold_m, threshold_n) - .with_context(|| { - format!("unknown committee for threshold_m={threshold_m} threshold_n={threshold_n}") - })? - .values() - .h, - ) -} - -/// Public-signal key for the aggregated PK commitment in `CircuitName::PkAggregation` (C5). -/// Must stay in lock-step with the Noir circuit's output ABI declaration. -const C5_PK_COMMITMENT_FIELD: &str = "commitment"; - -#[allow(clippy::too_many_arguments)] -pub(crate) fn verify_dkg_fold_attestation( - e3_id: &E3id, - party_id: u64, - proof: &Proof, - attestation: &SignedDkgFoldAttestation, - expected_node: &str, - committee_n: usize, - committee_h: usize, - n_moduli: usize, -) -> Result<()> { - ensure!( - attestation.payload.e3_id == *e3_id, - "attestation e3_id mismatch" - ); - ensure!( - attestation.payload.party_id == party_id, - "attestation party_id mismatch" - ); - let expected: Address = expected_node - .parse() - .with_context(|| format!("invalid committee node address {expected_node}"))?; - ensure!( - attestation.verify_signer(&expected)?, - "fold attestation signer does not match committee node for party {party_id}" - ); - let (extracted_party, commits) = - extract_node_fold_agg_commits(proof, committee_n, committee_h, n_moduli) - .map_err(|e| anyhow!("{e}"))?; - ensure!(extracted_party == party_id, "NodeFold party_id mismatch"); - ensure!( - commits == attestation.payload.agg_commits, - "NodeFold commits do not match signed attestation" - ); - Ok(()) -} - -/// Extract the hash-based aggregated PK commitment from the signed C5 proof. -/// This is the last public signal of `CircuitName::PkAggregation`. -pub(crate) fn extract_pk_commitment(c5_proof: &Proof) -> Result<[u8; 32]> { - let layout = CircuitName::PkAggregation.output_layout(); - let bytes = layout - .extract_field(&c5_proof.public_signals, C5_PK_COMMITMENT_FIELD) - .ok_or_else(|| anyhow::anyhow!("C5 proof is missing `commitment` public signal"))?; - let mut out = [0u8; 32]; - if bytes.len() != 32 { - return Err(anyhow::anyhow!( - "C5 `commitment` public signal must be 32 bytes" - )); - } - out.copy_from_slice(bytes); - Ok(out) -} - -/// Outcome of cross-checking each honest party's keyshare against its signed C1 -/// `pk_commitment` public signal. -pub(crate) struct C1CommitmentAudit { - /// Parties whose keyshare does not recompute to their signed C1 commitment, paired - /// with the proof for `SignedProofFailed` reporting. - pub mismatched: Vec<(u64, SignedProofPayload)>, - /// Parties that carried no C1 proof at all (defensive — normally already dishonest). - pub missing_proof: Vec, -} - -/// Recompute each honest party's `pk_commitment` from its keyshare bytes and compare it -/// against the `pk_commitment` public signal in the party's signed C1 proof. Pure: the -/// actor publishes `SignedProofFailed` for `mismatched` and folds both result sets into -/// `dishonest_parties`. -pub(crate) fn check_c1_keyshare_commitments( - entries: &[(u64, String, ArcBytes, Option)], - fhe: &Fhe, -) -> C1CommitmentAudit { - let mut mismatched = Vec::new(); - let mut missing_proof = Vec::new(); - for (party_id, _node, ks, c1) in entries { - let Some(signed_proof) = c1.as_ref() else { - warn!( - "Party {} has no C1 proof but was not marked dishonest", - party_id - ); - missing_proof.push(*party_id); - continue; - }; - let ok = match e3_zk_helpers::compute_pk_commitment_from_keyshare_bytes( - ks, - &fhe.params, - &fhe.crp, - ) { - Ok(computed) => signed_proof - .payload - .proof - .extract_output("pk_commitment") - .is_some_and(|extracted| extracted[..] == computed[..]), - Err(e) => { - warn!( - "Failed to compute pk_commitment for party {}: {}", - party_id, e - ); - false - } - }; - if !ok { - mismatched.push((*party_id, signed_proof.clone())); - } - } - C1CommitmentAudit { - mismatched, - missing_proof, - } -} - -#[allow(clippy::large_enum_variant)] -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub enum PublicKeyAggregatorState { - Collecting { - /// Live committee size after expulsions; used only to decide when enough keyshares arrived. - threshold_n: usize, - threshold_m: usize, - /// Canonical on-chain / circuit committee size N (unchanged after expulsion). - circuit_committee_n: usize, - /// Canonical honest-party count H for circuit public IO. - circuit_committee_h: usize, - keyshares: OrderedSet, - /// C1 proofs collected from KeyshareCreated events, indexed by insertion order - /// (matches `submission_order`). - c1_proofs: Vec>, - seed: Seed, - nodes: OrderedSet, - /// Insertion-ordered (real sortition `party_id`, node, keyshare) triples. - /// Index matches `c1_proofs`. The real `party_id` comes from `KeyshareCreated` - /// and must be used for all downstream circuit slot indexing — arrival order - /// is non-deterministic and does not match sortition's committee position. - submission_order: Vec<(u64, String, ArcBytes)>, - }, - VerifyingC1 { - /// Insertion-ordered (party_id, node, keyshare) triples from Collecting. - submission_order: Vec<(u64, String, ArcBytes)>, - threshold_m: usize, - /// Canonical on-chain / circuit committee size N (for `committee_h` lookup). - circuit_committee_n: usize, - /// Canonical honest-party count H for circuit public IO. - circuit_committee_h: usize, - /// C1 proofs in the same insertion order as `submission_order`. - c1_proofs: Vec>, - /// Real party_ids that submitted no C1 proof — treated as dishonest. - no_proof_parties: Vec, - }, - GeneratingC5Proof { - public_key: ArcBytes, - keyshare_bytes: Vec, - nodes: OrderedSet, - /// Registered node address per sortition `party_id` for the **full** committee - /// (all `N` parties that submitted a keyshare, honest or not). Honest-only lookups - /// must intersect with `honest_party_ids`. - party_nodes: HashMap, - /// DKG recursive proofs per party (restart-critical). - dkg_node_proofs: HashMap>, - /// Per-party fold attestations collected with honest DKG folds. - dkg_fold_attestations: HashMap, - honest_party_ids: BTreeSet, - dishonest_parties: BTreeSet, - /// Circuit committee size N (NodeFold / DKG public IO layout). - circuit_committee_n: usize, - /// Circuit honest-party count H (NodeFold / DKG public IO layout). - circuit_committee_h: usize, - /// In-flight [`ZkRequest::DkgAggregation`], if any. - dkg_aggregation_correlation: Option, - /// Result from [`ZkResponse::DkgAggregation`] (replaces pairwise `FoldProofs`). - dkg_aggregated_proof: Option, - c5_proof_pending: Option, - last_ec: Option>, - /// Accumulated nodes_fold proof after `nodes_fold_completed_slots` streaming steps. - nodes_fold_accumulator: Option, - /// Number of slots folded so far; equals the next slot index to dispatch. - nodes_fold_completed_slots: u32, - /// Correlation ID of the in-flight [`ZkRequest::NodesFoldStep`], if any. - nodes_fold_step_correlation: Option, - }, - Complete { - public_key: ArcBytes, - keyshares: OrderedSet, - nodes: OrderedSet, - /// Ascending `party_id` order (matches on-chain `topNodes` after finalize sort). - committee_addresses: Vec
, - /// Honest subset (H entries) for decryption-share gating after restart. - honest_committee_addresses: Vec
, - }, -} - -impl PublicKeyAggregatorState { - /// Ordered `topNodes` when the committee set is known (post–committee formation). - pub fn committee_nodes(&self) -> Option<&OrderedSet> { - match self { - PublicKeyAggregatorState::Collecting { nodes, .. } if !nodes.is_empty() => Some(nodes), - PublicKeyAggregatorState::GeneratingC5Proof { nodes, .. } => Some(nodes), - PublicKeyAggregatorState::Complete { nodes, .. } => Some(nodes), - _ => None, - } - } - - pub fn committee_addresses(&self) -> Option<&[Address]> { - match self { - PublicKeyAggregatorState::Complete { - committee_addresses, - .. - } if !committee_addresses.is_empty() => Some(committee_addresses.as_slice()), - _ => None, - } - } - - pub fn honest_committee_addresses(&self) -> Option<&[Address]> { - match self { - PublicKeyAggregatorState::Complete { - honest_committee_addresses, - .. - } if !honest_committee_addresses.is_empty() => { - Some(honest_committee_addresses.as_slice()) - } - _ => None, - } - } - - pub fn init(threshold_n: usize, threshold_m: usize, seed: Seed) -> Self { - let circuit_committee_h = committee_h_for(threshold_m, threshold_n) - .unwrap_or_else(|e| panic!("invalid committee at init: {e}")); - PublicKeyAggregatorState::Collecting { - threshold_n, - threshold_m, - circuit_committee_n: threshold_n, - circuit_committee_h, - keyshares: OrderedSet::new(), - c1_proofs: Vec::new(), - seed, - nodes: OrderedSet::new(), - submission_order: Vec::new(), - } - } -} - -/// Decision returned by [`PublicKeyAggregation::plan_c1_dispatch`]: which parties have a C1 -/// proof to verify and which submitted a keyshare without one (treated as dishonest). -pub(crate) struct C1Dispatch { - pub party_proofs: Vec, - pub no_proof_parties: Vec, -} - -/// Outcome of [`PublicKeyAggregation::select_honest_set`]. -pub(crate) enum HonestSelection { - /// Too few honest parties cleared C1 — caller must fail the E3. - Fail, - /// Enough honest parties — proceed to aggregation with this honest set. - Proceed { - honest_entries: Vec<(u64, String, ArcBytes, Option)>, - honest_party_ids: BTreeSet, - }, -} - -/// Plain, synchronous domain service for public-key aggregation decisions. -pub(crate) struct PublicKeyAggregation; - -impl PublicKeyAggregation { - /// Add a keyshare to a `Collecting` state, returning the next state. When all `N` - /// committee keyshares have arrived this transitions to `VerifyingC1`. Submitting a - /// `party_id` that already has a keyshare is idempotent (state is returned unchanged). - pub(crate) fn add_keyshare( - mut state: PublicKeyAggregatorState, - keyshare: ArcBytes, - node: String, - party_id: u64, - c1_proof: Option, - ) -> Result { - let PublicKeyAggregatorState::Collecting { - threshold_n, - threshold_m, - circuit_committee_n, - circuit_committee_h, - keyshares, - c1_proofs, - nodes, - submission_order, - .. - } = &mut state - else { - return Err(anyhow::anyhow!("Can only add keyshare in Collecting state")); - }; - - if submission_order.iter().any(|(pid, _, _)| *pid == party_id) { - return Ok(state); - } - - keyshares.insert(keyshare.clone()); - c1_proofs.push(c1_proof); - nodes.insert(node.clone()); - info!( - "add_keyshare: node={node} party_id={party_id} (arrival slot={})", - submission_order.len() - ); - submission_order.push((party_id, node, keyshare)); - let n = *threshold_n; - let m = *threshold_m; - let committee_n = *circuit_committee_n; - let committee_h = *circuit_committee_h; - let unique_parties = submission_order.len(); - info!( - "PublicKeyAggregator got keyshares {unique_parties}/{n} distinct parties (circuit_n={committee_n}, committee_h={committee_h})" - ); - // Collect all N committee keyshares before C1. C5 then requires exactly H honest - // proofs afterward (micro had N=H so waiting for H was equivalent). - if unique_parties >= n { - info!( - "Collected keyshares from {unique_parties} distinct parties (>= live_n={n}, circuit_n={committee_n}), transitioning to VerifyingC1..." - ); - return Ok(PublicKeyAggregatorState::VerifyingC1 { - submission_order: std::mem::take(submission_order), - threshold_m: m, - circuit_committee_n: committee_n, - circuit_committee_h: committee_h, - c1_proofs: std::mem::take(c1_proofs), - no_proof_parties: Vec::new(), - }); - } - - Ok(state) - } - - /// Split the collected keyshare submissions into parties with a C1 proof to verify and - /// parties that submitted no proof (treated as dishonest by the caller). - pub(crate) fn plan_c1_dispatch( - submission_order: &[(u64, String, ArcBytes)], - c1_proofs: &[Option], - ) -> C1Dispatch { - let mut party_proofs = Vec::new(); - let mut no_proof_parties = Vec::new(); - - for ((party_id, _, _), proof_opt) in submission_order.iter().zip(c1_proofs.iter()) { - match proof_opt { - Some(proof) => { - party_proofs.push(PartyProofsToVerify { - sender_party_id: *party_id, - signed_proofs: vec![proof.clone()], - }); - } - None => { - warn!( - "Party {} submitted keyshare without C1 proof — treating as dishonest", - party_id - ); - no_proof_parties.push(*party_id); - } - } - } - - C1Dispatch { - party_proofs, - no_proof_parties, - } - } - - /// Select the canonical honest set after C1 (ZK + commitment) filtering. - /// - /// Sorts the honest entries by real `party_id`, fails closed when fewer than `circuit_h` - /// parties cleared C1, caps the honest set to the `circuit_h` lowest `party_id`s, and - /// fails again if `<= threshold_m` parties remain. Logging mirrors the original handler. - pub(crate) fn select_honest_set( - e3_id: &E3id, - mut honest_entries: Vec<(u64, String, ArcBytes, Option)>, - dishonest_parties: &BTreeSet, - circuit_h: usize, - threshold_m: usize, - collected: usize, - ) -> HonestSelection { - // Sort by real party_id ascending so honest_keyshares / honest_nodes / - // honest_party_ids all share the same ordering used by NodeFold rows - // and by the circuit's slot indexing in `dkg_aggregator.nr`. - honest_entries.sort_by_key(|(pid, _, _, _)| *pid); - - if !dishonest_parties.is_empty() { - warn!( - "Total dishonest parties (ZK + commitment): {:?}", - dishonest_parties - ); - } - - // Fail closed when fewer than H parties cleared C1 — C5 cannot be witnessed. - if honest_entries.len() < circuit_h { - error!( - "C5 requires {circuit_h} honest parties with valid C1 proofs; only {} honest after verification (collected {collected}, dishonest: {:?})", - honest_entries.len(), - dishonest_parties - ); - return HonestSelection::Fail; - } - - // The C5 PkAggregation circuit is parameterised by a fixed honest-party count H. - // When more than H parties cleared C1, select the H lowest party_ids as the - // canonical honest set; the remainder stay in the full committee. - let pre_cap_len = honest_entries.len(); - let honest_party_ids = - cap_honest_party_ids(circuit_h, honest_entries.iter().map(|(pid, _, _, _)| *pid)); - if pre_cap_len > circuit_h { - info!( - "Capping honest set from {pre_cap_len} to circuit_h={circuit_h} for E3 {e3_id} (extras remain in full committee)" - ); - honest_entries.retain(|(pid, _, _, _)| honest_party_ids.contains(pid)); - } - - // Defensive: should hold after truncation above; guard against future refactors. - if honest_entries.len() <= threshold_m { - error!( - "Not enough honest parties after filtering: {} (need > {})", - honest_entries.len(), - threshold_m - ); - return HonestSelection::Fail; - } - - HonestSelection::Proceed { - honest_entries, - honest_party_ids, - } - } - - /// Apply a committee-member expulsion to a `Collecting` state, keeping the parallel - /// collections aligned and transitioning to `VerifyingC1` when enough keyshares remain. - pub(crate) fn handle_member_expelled( - mut state: PublicKeyAggregatorState, - node: &str, - ) -> Result { - let PublicKeyAggregatorState::Collecting { - threshold_n, - threshold_m, - circuit_committee_n, - circuit_committee_h, - keyshares, - c1_proofs, - nodes, - submission_order, - .. - } = &mut state - else { - return Ok(state); - }; - - let node_str = node.to_string(); - - // Find the expelled node's index in submission_order and remove from - // all parallel collections so they stay aligned. - if let Some(idx) = submission_order.iter().position(|(_, n, _)| n == &node_str) { - let (_, _, expelled_keyshare) = submission_order.remove(idx); - keyshares.remove(&expelled_keyshare); - c1_proofs.remove(idx); - } - - nodes.remove(&node_str); - - if *threshold_n > 0 { - *threshold_n -= 1; - info!( - "PublicKeyAggregator: reduced threshold_n to {} after expelling {}", - threshold_n, node - ); - } - - if *threshold_n < *threshold_m { - warn!( - "PublicKeyAggregator: threshold_n ({}) < threshold_m ({}) after expulsion — committee unviable", - threshold_n, threshold_m - ); - return Ok(state); - } - - if keyshares.len() == *threshold_n && *threshold_n > 0 { - let m = *threshold_m; - let committee_n = *circuit_committee_n; - let committee_h = *circuit_committee_h; - info!("PublicKeyAggregator: enough keyshares after expulsion, transitioning to VerifyingC1"); - return Ok(PublicKeyAggregatorState::VerifyingC1 { - submission_order: std::mem::take(submission_order), - threshold_m: m, - circuit_committee_n: committee_n, - circuit_committee_h: committee_h, - c1_proofs: std::mem::take(c1_proofs), - no_proof_parties: Vec::new(), - }); - } - - Ok(state) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn ks(byte: u8) -> ArcBytes { - ArcBytes::from_bytes(&[byte]) - } - - fn collecting(threshold_n: usize, threshold_m: usize) -> PublicKeyAggregatorState { - PublicKeyAggregatorState::init(threshold_n, threshold_m, Seed([0u8; 32])) - } - - #[test] - fn add_keyshare_below_threshold_stays_collecting() { - // minimum committee maps (m=1, n=3) -> needs 3 parties. - let state = collecting(3, 1); - let next = PublicKeyAggregation::add_keyshare(state, ks(1), "node-0".into(), 0, None) - .expect("add ok"); - match next { - PublicKeyAggregatorState::Collecting { - submission_order, .. - } => assert_eq!(submission_order.len(), 1), - _ => panic!("expected Collecting"), - } - } - - #[test] - fn add_keyshare_duplicate_party_is_idempotent() { - let state = collecting(3, 1); - let state = - PublicKeyAggregation::add_keyshare(state, ks(1), "node-0".into(), 0, None).unwrap(); - let state = - PublicKeyAggregation::add_keyshare(state, ks(9), "node-0".into(), 0, None).unwrap(); - match state { - PublicKeyAggregatorState::Collecting { - submission_order, - keyshares, - .. - } => { - assert_eq!(submission_order.len(), 1, "duplicate party ignored"); - assert_eq!(keyshares.len(), 1); - } - _ => panic!("expected Collecting"), - } - } - - #[test] - fn add_keyshare_reaching_threshold_transitions_to_verifying_c1() { - let mut state = collecting(3, 1); - for pid in 0..3u64 { - state = PublicKeyAggregation::add_keyshare( - state, - ks(pid as u8), - format!("node-{pid}"), - pid, - None, - ) - .unwrap(); - } - match state { - PublicKeyAggregatorState::VerifyingC1 { - submission_order, - circuit_committee_n, - circuit_committee_h, - threshold_m, - .. - } => { - assert_eq!(submission_order.len(), 3); - assert_eq!(circuit_committee_n, 3); - assert_eq!(circuit_committee_h, 2); - assert_eq!(threshold_m, 1); - } - _ => panic!("expected VerifyingC1"), - } - } - - #[test] - fn add_keyshare_wrong_state_errors() { - let state = PublicKeyAggregatorState::VerifyingC1 { - submission_order: vec![], - threshold_m: 1, - circuit_committee_n: 3, - circuit_committee_h: 2, - c1_proofs: vec![], - no_proof_parties: vec![], - }; - let err = PublicKeyAggregation::add_keyshare(state, ks(1), "n".into(), 0, None); - assert!(err.is_err()); - } - - #[test] - fn plan_c1_dispatch_splits_proofs_and_missing() { - let submission_order = vec![ - (0u64, "node-0".to_string(), ks(1)), - (1u64, "node-1".to_string(), ks(2)), - ]; - // party 0 has no proof, party 1 has one - let c1_proofs: Vec> = vec![None, None]; - let plan = PublicKeyAggregation::plan_c1_dispatch(&submission_order, &c1_proofs); - // both None -> both treated as no-proof - assert_eq!(plan.no_proof_parties, vec![0, 1]); - assert!(plan.party_proofs.is_empty()); - } - - #[test] - fn select_honest_set_fails_below_circuit_h() { - let e3_id = E3id::new("1", 1); - let honest = vec![(0u64, "n0".to_string(), ks(1), None)]; - let sel = - PublicKeyAggregation::select_honest_set(&e3_id, honest, &BTreeSet::new(), 3, 1, 1); - assert!(matches!(sel, HonestSelection::Fail)); - } - - #[test] - fn select_honest_set_caps_to_circuit_h_and_sorts() { - let e3_id = E3id::new("1", 1); - // 4 honest, circuit_h = 3 -> cap to lowest-3 party_ids {0,1,2}, sorted ascending. - let honest = vec![ - (3u64, "n3".to_string(), ks(4), None), - (1u64, "n1".to_string(), ks(2), None), - (0u64, "n0".to_string(), ks(1), None), - (2u64, "n2".to_string(), ks(3), None), - ]; - let sel = - PublicKeyAggregation::select_honest_set(&e3_id, honest, &BTreeSet::new(), 3, 1, 4); - match sel { - HonestSelection::Proceed { - honest_entries, - honest_party_ids, - } => { - let ids: Vec = honest_entries.iter().map(|(p, _, _, _)| *p).collect(); - assert_eq!(ids, vec![0, 1, 2]); - assert_eq!(honest_party_ids, BTreeSet::from([0, 1, 2])); - } - HonestSelection::Fail => panic!("expected Proceed"), - } - } - - #[test] - fn select_honest_set_fails_when_at_or_below_threshold_m() { - let e3_id = E3id::new("1", 1); - // 3 honest, circuit_h = 3 but threshold_m = 3 -> len <= m -> Fail. - let honest = vec![ - (0u64, "n0".to_string(), ks(1), None), - (1u64, "n1".to_string(), ks(2), None), - (2u64, "n2".to_string(), ks(3), None), - ]; - let sel = - PublicKeyAggregation::select_honest_set(&e3_id, honest, &BTreeSet::new(), 3, 3, 3); - assert!(matches!(sel, HonestSelection::Fail)); - } - - #[test] - fn handle_member_expelled_removes_and_reduces_threshold() { - let mut state = collecting(3, 1); - for pid in 0..2u64 { - state = PublicKeyAggregation::add_keyshare( - state, - ks(pid as u8), - format!("node-{pid}"), - pid, - None, - ) - .unwrap(); - } - // expel node-0; threshold_n 3 -> 2, keyshares now 1 (< 2) -> stays Collecting - let state = PublicKeyAggregation::handle_member_expelled(state, "node-0").unwrap(); - match state { - PublicKeyAggregatorState::Collecting { - threshold_n, - submission_order, - nodes, - .. - } => { - assert_eq!(threshold_n, 2); - assert_eq!(submission_order.len(), 1); - assert!(!nodes.contains(&"node-0".to_string())); - } - _ => panic!("expected Collecting"), - } - } - - #[test] - fn handle_member_expelled_transitions_when_enough_remain() { - // Collecting with n=2, m=1, two keyshares present; expel one -> - // threshold_n 1, keyshares 1 == n -> VerifyingC1. - let state = PublicKeyAggregatorState::Collecting { - threshold_n: 2, - threshold_m: 1, - circuit_committee_n: 3, - circuit_committee_h: 2, - keyshares: OrderedSet::from(vec![ks(10), ks(11)]), - c1_proofs: vec![None, None], - seed: Seed([0u8; 32]), - nodes: OrderedSet::from(vec!["node-0".to_string(), "node-1".to_string()]), - submission_order: vec![ - (0, "node-0".to_string(), ks(10)), - (1, "node-1".to_string(), ks(11)), - ], - }; - let next = PublicKeyAggregation::handle_member_expelled(state, "node-0").unwrap(); - match next { - PublicKeyAggregatorState::VerifyingC1 { - circuit_committee_n, - circuit_committee_h, - submission_order, - .. - } => { - assert_eq!(circuit_committee_n, 3); - assert_eq!(circuit_committee_h, 2); - assert_eq!(submission_order.len(), 1); - } - _ => panic!("expected VerifyingC1"), - } - } -} diff --git a/crates/aggregator/src/domain/threshold_plaintext_aggregation.rs b/crates/aggregator/src/domain/threshold_plaintext_aggregation.rs deleted file mode 100644 index 71f8fde386..0000000000 --- a/crates/aggregator/src/domain/threshold_plaintext_aggregation.rs +++ /dev/null @@ -1,645 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Plain, synchronous domain logic for threshold-plaintext (decryption) aggregation. -//! -//! This module holds the [`ThresholdPlaintextAggregatorState`] state machine plus the pure -//! transition/decision functions used by the `ThresholdPlaintextAggregator` actor. Nothing -//! here touches actix, `Persistable`, or the event bus: the actor feeds inputs in, gets a -//! next-state or a decision back, and performs the persistence/publish/dispatch side effects -//! itself. - -use std::collections::{BTreeMap, BTreeSet}; - -use anyhow::{bail, ensure, Result}; -use e3_events::CircuitName; -use e3_events::{ - DecryptionAggregationJobRequest, PartyProofsToVerify, Proof, Seed, SignedProofPayload, -}; -use e3_fhe_params::BfvPreset; -use e3_utils::utility_types::ArcBytes; -use e3_zk_helpers::circuits::commitments::compute_threshold_decryption_share_commitment; -use e3_zk_helpers::circuits::threshold::decrypted_shares_aggregation::MAX_MSG_NON_ZERO_COEFFS; -use e3_zk_helpers::threshold::share_decryption::{Bits as C6Bits, Bounds as C6Bounds}; -use e3_zk_helpers::Computation; -use tracing::{info, warn}; - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct Collecting { - pub(crate) threshold_m: u64, - pub(crate) threshold_n: u64, - pub(crate) shares: BTreeMap>, - /// Signed raw C6 proofs for ShareVerification. - pub(crate) c6_proofs: BTreeMap>, - pub(crate) seed: Seed, - pub(crate) ciphertext_output: Vec, - pub(crate) params: ArcBytes, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct VerifyingC6 { - pub(crate) threshold_m: u64, - pub(crate) threshold_n: u64, - pub(crate) shares: BTreeMap>, - pub(crate) c6_proofs: BTreeMap>, - pub(crate) ciphertext_output: Vec, - pub(crate) params: ArcBytes, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct Computing { - pub(crate) threshold_m: u64, - pub(crate) threshold_n: u64, - pub(crate) shares: Vec<(u64, Vec)>, - pub(crate) ciphertext_output: Vec, - pub(crate) params: ArcBytes, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct GeneratingC7Proof { - pub(crate) threshold_m: u64, - pub(crate) threshold_n: u64, - pub(crate) shares: Vec<(u64, Vec)>, - pub(crate) plaintext: Vec, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct Complete { - pub(crate) decrypted: Vec, - pub(crate) shares: Vec<(u64, Vec)>, -} - -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub enum ThresholdPlaintextAggregatorState { - Collecting(Collecting), - VerifyingC6(VerifyingC6), - Computing(Computing), - GeneratingC7Proof(GeneratingC7Proof), - Complete(Complete), -} - -impl TryFrom for Collecting { - type Error = anyhow::Error; - fn try_from( - value: ThresholdPlaintextAggregatorState, - ) -> std::result::Result { - match value { - ThresholdPlaintextAggregatorState::Collecting(s) => Ok(s), - _ => bail!("PlaintextState was expected to be Collecting but it was not."), - } - } -} - -impl TryFrom for VerifyingC6 { - type Error = anyhow::Error; - fn try_from( - value: ThresholdPlaintextAggregatorState, - ) -> std::result::Result { - match value { - ThresholdPlaintextAggregatorState::VerifyingC6(s) => Ok(s), - _ => bail!("Inconsistent state: expected VerifyingC6"), - } - } -} - -impl TryFrom for Computing { - type Error = anyhow::Error; - fn try_from( - value: ThresholdPlaintextAggregatorState, - ) -> std::result::Result { - match value { - ThresholdPlaintextAggregatorState::Computing(s) => Ok(s), - _ => bail!("Inconsistent state: expected Computing"), - } - } -} - -impl TryFrom for GeneratingC7Proof { - type Error = anyhow::Error; - fn try_from( - value: ThresholdPlaintextAggregatorState, - ) -> std::result::Result { - match value { - ThresholdPlaintextAggregatorState::GeneratingC7Proof(s) => Ok(s), - _ => bail!("Inconsistent state: expected GeneratingC7Proof"), - } - } -} - -impl TryFrom for Complete { - type Error = anyhow::Error; - fn try_from( - value: ThresholdPlaintextAggregatorState, - ) -> std::result::Result { - match value { - ThresholdPlaintextAggregatorState::Complete(s) => Ok(s), - _ => bail!("Inconsistent state: expected Complete"), - } - } -} - -impl ThresholdPlaintextAggregatorState { - pub fn init( - threshold_m: u64, - threshold_n: u64, - seed: Seed, - ciphertext_output: Vec, - params: ArcBytes, - ) -> Self { - ThresholdPlaintextAggregatorState::Collecting(Collecting { - threshold_m, - threshold_n, - shares: BTreeMap::new(), - c6_proofs: BTreeMap::new(), - seed, - ciphertext_output, - params, - }) - } -} - -/// Plain, synchronous domain service for threshold-plaintext aggregation decisions. -pub(crate) struct ThresholdPlaintextAggregation; - -impl ThresholdPlaintextAggregation { - /// Add a decryption share to a `Collecting` state, returning the next state. Once all - /// `required_shares` honest-committee shares have arrived this transitions to `VerifyingC6`. - /// `required_shares` is the canonical honest-committee size `H` (computed by the actor). - pub(crate) fn add_share( - state: ThresholdPlaintextAggregatorState, - party_id: u64, - share: Vec, - signed_decryption_proofs: Vec, - required_shares: u64, - ) -> Result { - info!("Adding share for party_id={}", party_id); - let current: Collecting = state.try_into()?; - let expected_outputs = current.ciphertext_output.len(); - ensure!( - share.len() == expected_outputs, - "party {party_id} supplied {} decryption shares for {expected_outputs} ciphertext outputs", - share.len() - ); - ensure!( - signed_decryption_proofs.len() == expected_outputs, - "party {party_id} supplied {} C6 proofs for {expected_outputs} ciphertext outputs", - signed_decryption_proofs.len() - ); - let ciphertext_output = current.ciphertext_output; - let threshold_m = current.threshold_m; - let threshold_n = current.threshold_n; - let params = current.params.clone(); - let mut shares = current.shares; - let mut c6_proofs = current.c6_proofs; - - info!("pushing to share collection {} {:?}", party_id, share); - shares.insert(party_id, share); - c6_proofs.insert(party_id, signed_decryption_proofs); - - if (shares.len() as u64) < required_shares { - return Ok(ThresholdPlaintextAggregatorState::Collecting(Collecting { - params, - threshold_n, - threshold_m, - ciphertext_output, - shares, - c6_proofs, - seed: current.seed, - })); - } - - info!( - "Changing state to VerifyingC6 because received all {required_shares} honest-committee shares..." - ); - - Ok(ThresholdPlaintextAggregatorState::VerifyingC6( - VerifyingC6 { - shares, - c6_proofs, - ciphertext_output, - threshold_m, - threshold_n, - params, - }, - )) - } - - /// Apply a committee-member expulsion to a `Collecting` state, removing the party's share - /// and C6 proofs, and transitioning to `VerifyingC6` when enough shares remain. - pub(crate) fn handle_member_expelled( - state: ThresholdPlaintextAggregatorState, - party_id: u64, - required_shares: u64, - ) -> Result { - let ThresholdPlaintextAggregatorState::Collecting(current) = state else { - return Ok(state); - }; - - let mut shares = current.shares; - let mut c6_proofs = current.c6_proofs; - let threshold_n = current.threshold_n; - - shares.remove(&party_id); - c6_proofs.remove(&party_id); - - if required_shares < current.threshold_m { - warn!( - "ThresholdPlaintextAggregator: honest committee size H ({required_shares}) < threshold_m ({}) after expulsion", - current.threshold_m - ); - return Ok(ThresholdPlaintextAggregatorState::Collecting(Collecting { - threshold_m: current.threshold_m, - threshold_n, - shares, - c6_proofs, - seed: current.seed, - ciphertext_output: current.ciphertext_output, - params: current.params, - })); - } - - if (shares.len() as u64) < required_shares { - return Ok(ThresholdPlaintextAggregatorState::Collecting(Collecting { - threshold_m: current.threshold_m, - threshold_n, - shares, - c6_proofs, - seed: current.seed, - ciphertext_output: current.ciphertext_output, - params: current.params, - })); - } - - Ok(ThresholdPlaintextAggregatorState::VerifyingC6( - VerifyingC6 { - threshold_m: current.threshold_m, - threshold_n, - shares, - c6_proofs, - ciphertext_output: current.ciphertext_output, - params: current.params, - }, - )) - } - - /// Build the per-party C6 proof bundles dispatched to ShareVerification. - pub(crate) fn plan_c6_dispatch( - c6_proofs: BTreeMap>, - ) -> Vec { - c6_proofs - .into_iter() - .map(|(party_id, signed_proofs)| PartyProofsToVerify { - sender_party_id: party_id, - signed_proofs, - }) - .collect() - } - - /// Verify that each honest party's raw decryption share bytes match the - /// `d_commitment` output in their verified C6 proof. Returns party IDs - /// that failed the check. - /// - /// Catches the attack where a node sends a valid C6 proof for share `d_A` but - /// broadcasts different bytes `d_B`. - pub(crate) fn verify_shares_match_c6_commitments( - params_preset: BfvPreset, - honest_shares: &[(u64, Vec)], - c6_proofs: &BTreeMap>, - ) -> BTreeSet { - let mut mismatched = BTreeSet::new(); - - let Ok((threshold_params, _)) = e3_fhe_params::build_pair_for_preset(params_preset) else { - warn!("Could not build BFV params for d_commitment check — skipping"); - return mismatched; - }; - - // Reuse the same Bounds/Bits computation that C6 codegen uses, - // so d_native_bit stays in sync if the formula ever changes. - let Ok(bounds) = C6Bounds::compute(params_preset, &()) else { - warn!("Could not compute bounds for d_commitment check — skipping"); - return mismatched; - }; - let Ok(bits) = C6Bits::compute(params_preset, &bounds) else { - warn!("Could not compute bits for d_commitment check — skipping"); - return mismatched; - }; - let d_native_bit = bits.d_native_bit; - - let max_k = MAX_MSG_NON_ZERO_COEFFS; - let c6_output_layout = CircuitName::ThresholdShareDecryption.output_layout(); - - for (party_id, shares) in honest_shares { - let Some(proofs) = c6_proofs.get(party_id) else { - warn!( - "No C6 proofs for party {} — marking as mismatched", - party_id - ); - mismatched.insert(*party_id); - continue; - }; - let Some(first_proof) = proofs.first() else { - warn!( - "Empty C6 proof list for party {} — marking as mismatched", - party_id - ); - mismatched.insert(*party_id); - continue; - }; - let Some(c6_d_bytes) = c6_output_layout - .extract_field(&first_proof.payload.proof.public_signals, "d_commitment") - else { - warn!( - "Could not extract d_commitment from C6 proof for party {} — marking as mismatched", - party_id - ); - mismatched.insert(*party_id); - continue; - }; - - let Some(share_bytes) = shares.first() else { - warn!( - "No share bytes for party {} — marking as mismatched", - party_id - ); - mismatched.insert(*party_id); - continue; - }; - let Ok(poly) = - e3_trbfv::helpers::try_poly_pb_from_bytes(share_bytes, &threshold_params) - else { - warn!( - "Could not deserialize share for party {} — marking as mismatched", - party_id - ); - mismatched.insert(*party_id); - continue; - }; - let crt = e3_polynomial::CrtPolynomial::from_fhe_polynomial(&poly); - - // C6 public `d_commitment` hashes native truncated limbs (same layout as C7), not - // reversed+centered witness `d`. - let computed = compute_threshold_decryption_share_commitment(&crt, d_native_bit, max_k); - - // Convert to big-endian 32-byte padded format matching - // Barretenberg's public_signals encoding. - let (_, be_bytes) = computed.to_bytes_be(); - let mut computed_padded = [0u8; 32]; - let start = 32usize.saturating_sub(be_bytes.len()); - computed_padded[start..].copy_from_slice(&be_bytes[..be_bytes.len().min(32)]); - - if computed_padded != c6_d_bytes { - warn!( - "d_commitment mismatch for party {}: raw share commitment differs from C6 proof output", - party_id - ); - mismatched.insert(*party_id); - } - } - - mismatched - } -} - -/// Pad/truncate each decrypted plaintext limb to the fixed `MAX_MSG_NON_ZERO_COEFFS * 8` -/// byte width expected by consumers of `PlaintextAggregated`. -pub(crate) fn format_decrypted_plaintext(plaintext: &[ArcBytes]) -> Vec { - let len = MAX_MSG_NON_ZERO_COEFFS * 8; - plaintext - .iter() - .map(|pt| { - let mut bytes = pt.extract_bytes(); - if bytes.len() >= len { - bytes.truncate(len); - } else { - bytes.resize(len, 0); - } - ArcBytes::from_bytes(&bytes) - }) - .collect() -} - -/// Bind each C7 (per-ciphertext) proof to the first `c6_total_slots` honest C6 inner -/// proofs for that ciphertext, producing the per-ciphertext decryption-aggregation jobs. -/// Returns `None` when an expected C6 inner proof is missing for some ciphertext index -/// (the actor then fails the decryption round). -pub(crate) fn build_decryption_aggregation_jobs( - c7_proofs: &[Proof], - honest_c6: &[(u64, Vec)], - c6_total_slots: usize, -) -> Option> { - let mut jobs = Vec::with_capacity(c7_proofs.len()); - for (ct_idx, c7_proof) in c7_proofs.iter().enumerate() { - let mut c6_inner_proofs = Vec::with_capacity(c6_total_slots); - let c6_slot_indices: Vec = (0..c6_total_slots as u32).collect(); - for (_, wps) in honest_c6.iter().take(c6_total_slots) { - let Some(p) = wps.get(ct_idx) else { - warn!("C6 inner proof missing for party at ct index {}", ct_idx); - return None; - }; - c6_inner_proofs.push(p.clone()); - } - jobs.push(DecryptionAggregationJobRequest { - c6_inner_proofs, - c6_slot_indices, - c7_proof: c7_proof.clone(), - }); - } - Some(jobs) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn ab(b: u8) -> ArcBytes { - ArcBytes::from_bytes(&[b]) - } - - fn c6_proof(marker: u8) -> SignedProofPayload { - SignedProofPayload { - payload: e3_events::ProofPayload { - e3_id: e3_events::E3id::new("1", 1), - proof_type: e3_events::ProofType::C6ThresholdShareDecryption, - proof: Proof::new( - CircuitName::ThresholdShareDecryption, - ab(marker), - ab(marker), - ), - }, - signature: ab(marker), - } - } - - fn collecting(threshold_m: u64, threshold_n: u64) -> ThresholdPlaintextAggregatorState { - ThresholdPlaintextAggregatorState::init( - threshold_m, - threshold_n, - Seed([0u8; 32]), - vec![ab(1)], - ab(2), - ) - } - - #[test] - fn add_share_below_required_stays_collecting() { - let state = collecting(1, 3); - let next = - ThresholdPlaintextAggregation::add_share(state, 0, vec![ab(10)], vec![c6_proof(10)], 3) - .unwrap(); - match next { - ThresholdPlaintextAggregatorState::Collecting(c) => { - assert_eq!(c.shares.len(), 1); - assert!(c.shares.contains_key(&0)); - } - _ => panic!("expected Collecting"), - } - } - - #[test] - fn add_share_reaching_required_transitions_to_verifying_c6() { - let mut state = collecting(1, 3); - for pid in 0..3u64 { - state = ThresholdPlaintextAggregation::add_share( - state, - pid, - vec![ab(pid as u8)], - vec![c6_proof(pid as u8)], - 3, - ) - .unwrap(); - } - match state { - ThresholdPlaintextAggregatorState::VerifyingC6(v) => { - assert_eq!(v.shares.len(), 3); - } - _ => panic!("expected VerifyingC6"), - } - } - - #[test] - fn add_share_wrong_state_errors() { - let state = ThresholdPlaintextAggregatorState::VerifyingC6(VerifyingC6 { - threshold_m: 1, - threshold_n: 3, - shares: BTreeMap::new(), - c6_proofs: BTreeMap::new(), - ciphertext_output: vec![ab(1)], - params: ab(2), - }); - let res = ThresholdPlaintextAggregation::add_share(state, 0, vec![ab(0)], vec![], 3); - assert!(res.is_err()); - } - - #[test] - fn handle_member_expelled_removes_share_and_stays_collecting() { - let mut state = collecting(1, 3); - for pid in 0..2u64 { - state = ThresholdPlaintextAggregation::add_share( - state, - pid, - vec![ab(pid as u8)], - vec![c6_proof(pid as u8)], - 3, - ) - .unwrap(); - } - // required_shares stays 3; remove party 0 -> 1 share left -> Collecting - let next = ThresholdPlaintextAggregation::handle_member_expelled(state, 0, 3).unwrap(); - match next { - ThresholdPlaintextAggregatorState::Collecting(c) => { - assert_eq!(c.shares.len(), 1); - assert!(!c.shares.contains_key(&0)); - } - _ => panic!("expected Collecting"), - } - } - - #[test] - fn handle_member_expelled_transitions_when_enough_remain() { - let mut state = collecting(1, 3); - for pid in 0..3u64 { - state = ThresholdPlaintextAggregation::add_share( - state, - pid, - vec![ab(pid as u8)], - vec![c6_proof(pid as u8)], - 3, - ) - .unwrap(); - } - // After 3 shares it's already VerifyingC6; rebuild a Collecting with 3 shares to - // exercise the expulsion->VerifyingC6 path with required_shares lowered to 2. - let state = ThresholdPlaintextAggregatorState::Collecting(Collecting { - threshold_m: 1, - threshold_n: 3, - shares: BTreeMap::from([(0, vec![ab(0)]), (1, vec![ab(1)]), (2, vec![ab(2)])]), - c6_proofs: BTreeMap::new(), - seed: Seed([0u8; 32]), - ciphertext_output: vec![ab(1)], - params: ab(2), - }); - let _ = state; - let state = ThresholdPlaintextAggregatorState::Collecting(Collecting { - threshold_m: 1, - threshold_n: 3, - shares: BTreeMap::from([(0, vec![ab(0)]), (1, vec![ab(1)]), (2, vec![ab(2)])]), - c6_proofs: BTreeMap::new(), - seed: Seed([0u8; 32]), - ciphertext_output: vec![ab(1)], - params: ab(2), - }); - // remove party 0 -> 2 shares remain, required_shares=2 -> VerifyingC6 - let next = ThresholdPlaintextAggregation::handle_member_expelled(state, 0, 2).unwrap(); - match next { - ThresholdPlaintextAggregatorState::VerifyingC6(v) => { - assert_eq!(v.shares.len(), 2); - } - _ => panic!("expected VerifyingC6"), - } - } - - #[test] - fn handle_member_expelled_wrong_state_is_noop() { - let state = ThresholdPlaintextAggregatorState::Complete(Complete { - decrypted: vec![ab(1)], - shares: vec![], - }); - let next = ThresholdPlaintextAggregation::handle_member_expelled(state, 0, 3).unwrap(); - assert!(matches!( - next, - ThresholdPlaintextAggregatorState::Complete(_) - )); - } - - #[test] - fn add_share_rejects_c6_share_or_proof_count_mismatch() { - let missing_share = ThresholdPlaintextAggregation::add_share( - collecting(1, 3), - 0, - vec![], - vec![c6_proof(1)], - 3, - ) - .expect_err("one decryption share is required for one ciphertext"); - assert!(missing_share.to_string().contains("decryption shares")); - - let missing_proof = - ThresholdPlaintextAggregation::add_share(collecting(1, 3), 0, vec![ab(1)], vec![], 3) - .expect_err("one C6 proof is required for one ciphertext"); - assert!(missing_proof.to_string().contains("C6 proofs")); - } - - #[test] - fn plan_c6_dispatch_emits_party_proofs_in_party_order() { - let mut c6: BTreeMap> = BTreeMap::new(); - c6.insert(2, vec![]); - c6.insert(0, vec![]); - c6.insert(1, vec![]); - let plan = ThresholdPlaintextAggregation::plan_c6_dispatch(c6); - let ids: Vec = plan.iter().map(|p| p.sender_party_id).collect(); - assert_eq!(ids, vec![0, 1, 2]); - } -} diff --git a/crates/aggregator/src/domain/failover.rs b/crates/aggregator/src/failover.rs similarity index 100% rename from crates/aggregator/src/domain/failover.rs rename to crates/aggregator/src/failover.rs diff --git a/crates/aggregator/src/lib.rs b/crates/aggregator/src/lib.rs index 8f73507055..150c387fba 100644 --- a/crates/aggregator/src/lib.rs +++ b/crates/aggregator/src/lib.rs @@ -6,16 +6,16 @@ //! Public-key and threshold-plaintext aggregation. //! -//! The crate is organised into two layers: -//! - [`actors`] — thin actix actors that own persistence and the event bus and -//! route messages between the protocol and the domain services. -//! - [`domain`] — pure, synchronous services holding the aggregation state -//! machines and cryptographic combination logic, unit-tested in isolation. +//! Implementation is grouped by the `committee_finalization`, +//! `public_key_aggregation`, and `plaintext_aggregation` capabilities. The private +//! [`actors`], [`workflow`], and [`domain`] modules are compatibility views that +//! preserve established Rust paths; they do not define the filesystem layout. mod actors; mod domain; pub mod ext; mod repo; +mod workflow; pub use actors::*; pub use domain::committee_hash; diff --git a/crates/aggregator/src/plaintext_aggregation/actor.rs b/crates/aggregator/src/plaintext_aggregation/actor.rs new file mode 100644 index 0000000000..74e23d3c7c --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/actor.rs @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::collections::{BTreeMap, BTreeSet}; +use std::str::FromStr; +use std::time::Duration; + +use crate::workflow::threshold_plaintext_aggregation::{ + build_decryption_aggregation_jobs, format_decrypted_plaintext, ThresholdPlaintextAggregation, +}; +use actix::prelude::*; +use actix::SpawnHandle; +use alloy::primitives::Address; +use anyhow::{anyhow, bail, ensure, Result}; +use e3_data::Persistable; +use e3_events::{ + prelude::*, trap, AggregationProofPending, AggregationProofSigned, BusHandle, + CommitteeMemberExpelled, ComputeRequest, ComputeRequestError, ComputeRequestErrorKind, + ComputeResponse, ComputeResponseKind, CorrelationId, DecryptedSharesAggregationProofRequest, + DecryptionAggregationRequest, DecryptionshareCreated, Die, E3Failed, E3Stage, E3id, EType, + EventContext, FailureReason, InterfoldEvent, InterfoldEventData, PlaintextAggregated, Proof, + Sequenced, ShareVerificationComplete, ShareVerificationDispatched, SignedProofPayload, + TypedEvent, VerificationKind, ZkRequest, ZkResponse, +}; +use e3_fhe_params::BfvPreset; +use e3_sortition::{E3CommitteeContainsRequest, E3CommitteeContainsResponse, Sortition}; +use e3_trbfv::{ + calculate_threshold_decryption::CalculateThresholdDecryptionRequest, TrBFVConfig, TrBFVRequest, + TrBFVResponse, +}; +use e3_utils::NotifySync; +use e3_utils::{utility_types::ArcBytes, MAILBOX_LIMIT}; +use e3_zk_helpers::CiphernodesCommitteeSize; +use tracing::{debug, info, trace, warn}; + +/// Env var overriding the decryption-share collection timeout (seconds). +const DECRYPTION_COLLECTION_TIMEOUT_ENV: &str = "E3_DECRYPTION_COLLECTION_TIMEOUT_SECS"; +/// Default wall-clock budget for collecting the honest committee's decryption shares before the +/// round is failed loudly. Without this bound a single absent honest member stalls the decryption +/// round forever (the collector waits for all `H` honest shares with no fallback). +const DEFAULT_DECRYPTION_COLLECTION_TIMEOUT_SECS: u64 = 1800; + +/// Resolve the decryption-share collection timeout, honouring the env override. +fn decryption_collection_timeout() -> Duration { + match std::env::var(DECRYPTION_COLLECTION_TIMEOUT_ENV) + .ok() + .and_then(|v| v.parse::().ok()) + { + Some(secs) => { + info!( + "Decryption-share collection timeout overridden via {}={}s", + DECRYPTION_COLLECTION_TIMEOUT_ENV, secs + ); + Duration::from_secs(secs) + } + None => Duration::from_secs(DEFAULT_DECRYPTION_COLLECTION_TIMEOUT_SECS), + } +} + +/// Internal self-message fired when the decryption-share collection window elapses. +#[derive(Message, Clone, Debug)] +#[rtype(result = "()")] +struct DecryptionCollectionTimeout; + +// Threshold-plaintext aggregation state machine + pure transition logic now live in +// `crate::workflow::threshold_plaintext_aggregation`; re-exported here to preserve the public path +// `e3_aggregator::threshold_plaintext_aggregator::*` (and the crate-level glob re-export). +pub use crate::workflow::threshold_plaintext_aggregation::{ + Collecting, Complete, Computing, GeneratingC7Proof, ThresholdPlaintextAggregatorState, + VerifyingC6, +}; + +/// Process-local effect state. Persisted protocol progression remains in +/// `ThresholdPlaintextAggregatorState`; these values are reconstructed or +/// redriven from replayed facts. +#[derive(Default)] +struct PendingDecryptionWork { + /// Honest parties' C6 inner proofs (sorted by party id) for [`ZkRequest::DecryptionAggregation`]. + honest_c6_proofs_for_agg: Option)>>, + /// In-flight threshold decryption request. + threshold_decryption_correlation: Option, + /// In-flight decryption aggregation request. + decryption_aggregation_correlation: Option, + /// C7 proofs stored while waiting for decryption aggregation. + c7_proofs_pending: Option>, + /// DecryptionAggregator outputs (set when ZK completes). + decryption_aggregator_proofs: Option>, + /// Last event context, reused for ZK and final publish. + last_ec: Option>, + /// Timer handle for the decryption-share collection timeout (cancelled when the actor stops). + timeout_handle: Option, + /// Most recent inbound event context, used as the causal parent for the `E3Failed` event + /// emitted if the collection window elapses while still collecting shares. + timeout_ec: Option>, +} + +pub struct ThresholdPlaintextAggregator { + bus: BusHandle, + sortition: Addr, + e3_id: E3id, + params_preset: BfvPreset, + committee_size: CiphernodesCommitteeSize, + proof_aggregation_enabled: bool, + state: Persistable, + /// Full registered committee (`topNodes`, length `N`) for decryption-aggregator + /// `committee_hash_*` inputs. Same value as `PublicKeyAggregated.committee_addresses`. + committee_addresses: Vec
, + /// Canonical honest subset from DKG (length `H ≤ N`, from + /// `PublicKeyAggregated.honest_committee_addresses`). Drives share-collection + /// gating (expects one share from each H party) and sender checks after sortition. + honest_committee_addresses: Vec
, + pending: PendingDecryptionWork, +} + +pub struct ThresholdPlaintextAggregatorParams { + pub bus: BusHandle, + pub sortition: Addr, + pub e3_id: E3id, + pub params_preset: BfvPreset, + pub committee_size: CiphernodesCommitteeSize, + pub proof_aggregation_enabled: bool, + /// Full committee from `PublicKeyAggregated.committee_addresses` (length `N`). + /// Used for `committee_hash_*` payload binding to on-chain `topNodes`. + pub committee_addresses: Vec
, + /// Honest committee from `PublicKeyAggregated.honest_committee_addresses` + /// (length `H`). Roster for decryption-share collection and sender gating. + pub honest_committee_addresses: Vec
, +} + +fn node_owns_committee_party_slot( + committee: &[Address], + honest_committee: &[Address], + node: &str, + party_id: u64, +) -> bool { + let Some(expected) = usize::try_from(party_id) + .ok() + .and_then(|index| committee.get(index)) + else { + return false; + }; + Address::from_str(node) + .ok() + .is_some_and(|address| &address == expected && honest_committee.contains(&address)) +} + +impl ThresholdPlaintextAggregator { + pub fn new( + params: ThresholdPlaintextAggregatorParams, + state: Persistable, + ) -> Self { + ThresholdPlaintextAggregator { + bus: params.bus, + sortition: params.sortition, + e3_id: params.e3_id, + params_preset: params.params_preset, + committee_size: params.committee_size, + proof_aggregation_enabled: params.proof_aggregation_enabled, + state, + committee_addresses: params.committee_addresses, + honest_committee_addresses: params.honest_committee_addresses, + pending: PendingDecryptionWork::default(), + } + } + + /// Length of the canonical honest subset (`H`), not on-chain committee size `N`. + /// Share collection waits for one decryption share from each address in + /// `honest_committee_addresses` (sortition membership is checked separately). + fn aggregated_committee_n(&self) -> u64 { + self.honest_committee_addresses.len() as u64 + } + + /// True when `node` owns `party_id` in the full canonical committee and is part of the honest + /// subset selected during DKG. Membership without the slot check permits a real member to + /// relabel a share under another party ID. + fn node_owns_aggregated_pk_party_slot(&self, node: &str, party_id: u64) -> bool { + node_owns_committee_party_slot( + &self.committee_addresses, + &self.honest_committee_addresses, + node, + party_id, + ) + } +} + +#[path = "effects/mod.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +#[cfg(test)] +#[path = "tests/mod.rs"] +mod tests; diff --git a/crates/aggregator/src/actors/decryptionshare_created_buffer.rs b/crates/aggregator/src/plaintext_aggregation/decryption_share_buffer.rs similarity index 100% rename from crates/aggregator/src/actors/decryptionshare_created_buffer.rs rename to crates/aggregator/src/plaintext_aggregation/decryption_share_buffer.rs diff --git a/crates/aggregator/src/plaintext_aggregation/effects/mod.rs b/crates/aggregator/src/plaintext_aggregation/effects/mod.rs new file mode 100644 index 0000000000..fd7b9d843b --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/effects/mod.rs @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Effect execution for threshold-plaintext aggregation. + +use super::*; + +mod prove_plaintext; +mod publish_result; +mod verify_decryption_shares; diff --git a/crates/aggregator/src/plaintext_aggregation/effects/prove_plaintext.rs b/crates/aggregator/src/plaintext_aggregation/effects/prove_plaintext.rs new file mode 100644 index 0000000000..3b796f0723 --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/effects/prove_plaintext.rs @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Threshold decryption, C7 proving, and recursive decryption aggregation. + +use super::*; + +impl ThresholdPlaintextAggregator { + /// Publish AggregationProofPending for C7 proof generation through ProofRequestActor. + pub fn dispatch_c7_proof_request( + &mut self, + shares: Vec<(u64, Vec)>, + plaintext: Vec, + threshold_m: u64, + threshold_n: u64, + ec: EventContext, + ) -> Result<()> { + self.bus.publish( + AggregationProofPending { + e3_id: self.e3_id.clone(), + proof_request: DecryptedSharesAggregationProofRequest { + d_share_polys: shares.clone(), + plaintext: plaintext.clone(), + params_preset: self.params_preset, + threshold_m, + threshold_n, + committee_size: self.committee_size, + }, + plaintext, + shares, + }, + ec, + )?; + Ok(()) + } + + /// Handle AggregationProofSigned: store C7 proofs and wait for C6 fold before publishing. + pub fn handle_aggregation_proof_signed( + &mut self, + msg: TypedEvent, + _ctx: &mut Context, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + + if msg.e3_id != self.e3_id { + return Ok(()); + } + + let state: GeneratingC7Proof = self + .state + .get() + .ok_or(anyhow!("Could not get state"))? + .try_into()?; + + // Extract raw proofs from signed payloads for PlaintextAggregated + let proofs: Vec<_> = msg + .signed_proofs + .iter() + .map(|sp| sp.payload.proof.clone()) + .collect(); + + if proofs.len() != state.plaintext.len() { + warn!( + "C7 proof count mismatch: got {} proofs for {} ciphertext indices", + proofs.len(), + state.plaintext.len() + ); + return self.fail_decryption_round(ec); + } + + info!("C7 proof signed — awaiting DecryptionAggregation..."); + self.pending.c7_proofs_pending = Some(proofs); + self.pending.last_ec = Some(ec.clone()); + self.maybe_start_decryption_aggregation(&ec)?; + self.try_publish_complete() + } + + pub(in crate::actors::threshold_plaintext_aggregator) fn maybe_start_decryption_aggregation( + &mut self, + ec: &EventContext, + ) -> Result<()> { + if self.pending.c7_proofs_pending.is_none() { + return Ok(()); + } + if self.pending.decryption_aggregator_proofs.is_some() + || self.pending.decryption_aggregation_correlation.is_some() + { + return Ok(()); + } + if !self.proof_aggregation_enabled { + if self.pending.decryption_aggregator_proofs.is_none() { + self.pending.decryption_aggregator_proofs = Some(Vec::new()); + } + return Ok(()); + } + self.dispatch_decryption_aggregation(ec) + } + + pub(in crate::actors::threshold_plaintext_aggregator) fn dispatch_decryption_aggregation( + &mut self, + ec: &EventContext, + ) -> Result<()> { + if self.committee_addresses.is_empty() { + warn!( + e3_id = %self.e3_id, + "DecryptionAggregation: committee addresses missing at aggregator construction" + ); + return self.fail_decryption_round(ec.clone()); + } + + let Some(c7_proofs) = self.pending.c7_proofs_pending.as_ref() else { + return Ok(()); + }; + if self.pending.decryption_aggregator_proofs.is_some() { + return Ok(()); + } + if self.pending.decryption_aggregation_correlation.is_some() { + return Ok(()); + } + if !self.proof_aggregation_enabled { + self.pending.decryption_aggregator_proofs = Some(Vec::new()); + return Ok(()); + } + let Some(honest_c6) = self.pending.honest_c6_proofs_for_agg.as_ref() else { + warn!( + e3_id = %self.e3_id, + "DecryptionAggregation deferred: honest C6 proofs not retained on aggregator" + ); + return Ok(()); + }; + // With proof aggregation enabled we must have a complete C6 set; otherwise we'd publish + // `decryption_aggregator_proofs = Vec::new()`, which downstream consumers interpret as + // "aggregation disabled". Fail loudly instead so the missing shares are surfaced. + if honest_c6.is_empty() || honest_c6.iter().any(|(_, w)| w.is_empty()) { + warn!( + "DecryptionAggregation: honest C6 inner proofs missing while proof aggregation is enabled" + ); + return self.fail_decryption_round(ec.clone()); + } + let state: GeneratingC7Proof = self + .state + .get() + .ok_or(anyhow!("Could not get state"))? + .try_into()?; + // C6Fold witness width is `T + 1` (same `T` as `threshold_m`). C7 is only proven for the + // first `T + 1` parties after sorting by party id (`handle_decrypted_shares_aggregation_proof` + // truncates); fold slot indices must stay in `0..T+1` and use that same party subset. + let c6_total_slots = state.threshold_m as usize + 1; + if honest_c6.len() < c6_total_slots { + warn!( + "DecryptionAggregation needs at least {} honest C6 parties, have {}", + c6_total_slots, + honest_c6.len() + ); + return self.fail_decryption_round(ec.clone()); + } + let num_ct = c7_proofs.len(); + let Some(jobs) = build_decryption_aggregation_jobs(c7_proofs, honest_c6, c6_total_slots) + else { + return self.fail_decryption_round(ec.clone()); + }; + let corr = CorrelationId::new(); + info!( + e3_id = %self.e3_id, + num_jobs = num_ct, + c6_slots = c6_total_slots, + "DecryptionAggregation: publishing Zk compute request" + ); + self.bus.publish( + ComputeRequest::zk( + ZkRequest::DecryptionAggregation(DecryptionAggregationRequest { + c6_total_slots, + jobs, + committee_addresses: self.committee_addresses.clone(), + params_preset: self.params_preset, + committee_size: self.committee_size, + }), + corr, + self.e3_id.clone(), + ), + ec.clone(), + )?; + self.pending.decryption_aggregation_correlation = Some(corr); + Ok(()) + } + + pub fn handle_compute_response( + &mut self, + msg: TypedEvent, + ctx: &mut Context, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + ensure!( + msg.e3_id == self.e3_id, + "PlaintextAggregator should never receive incorrect e3_id msgs" + ); + + let correlation_id = msg.correlation_id; + match msg.response { + // TrBFV threshold decryption response -> transition to GeneratingC7Proof + ComputeResponseKind::TrBFV(TrBFVResponse::CalculateThresholdDecryption(response)) => { + if self.pending.threshold_decryption_correlation.as_ref() != Some(&correlation_id) { + return Ok(()); + } + self.pending.threshold_decryption_correlation = None; + info!("Received TrBFV threshold decryption response"); + let plaintext = response.plaintext; + + let state: Computing = self + .state + .get() + .ok_or(anyhow!("Could not get state"))? + .try_into()?; + + let shares = state.shares.clone(); + let threshold_m = state.threshold_m; + let threshold_n = state.threshold_n; + + // Publish pending event before transitioning state so a publish + // failure leaves us in Computing (retryable) rather than + // GeneratingC7Proof (no retry path). + self.dispatch_c7_proof_request( + shares.clone(), + plaintext.clone(), + threshold_m, + threshold_n, + ec.clone(), + )?; + + // Transition to GeneratingC7Proof + self.state.try_mutate(&ec, |_| { + Ok(ThresholdPlaintextAggregatorState::GeneratingC7Proof( + GeneratingC7Proof { + threshold_m, + threshold_n, + shares, + plaintext, + }, + )) + })?; + } + + ComputeResponseKind::Zk(ZkResponse::DecryptionAggregation(resp)) => { + if self.pending.decryption_aggregation_correlation.as_ref() == Some(&correlation_id) + { + self.pending.decryption_aggregation_correlation = None; + // Worker must return one DecryptionAggregator proof per pending C7 ciphertext. + if let Some(c7_proofs) = self.pending.c7_proofs_pending.as_ref() { + if resp.proofs.len() != c7_proofs.len() { + warn!( + "DecryptionAggregation response proof count {} != expected {}", + resp.proofs.len(), + c7_proofs.len() + ); + return self.fail_decryption_round(ec); + } + } + self.pending.decryption_aggregator_proofs = Some(resp.proofs); + self.try_publish_complete()?; + } + } + + _ => { + // Not a response we handle — ignore + } + } + let _ = ctx; + Ok(()) + } +} diff --git a/crates/aggregator/src/plaintext_aggregation/effects/publish_result.rs b/crates/aggregator/src/plaintext_aggregation/effects/publish_result.rs new file mode 100644 index 0000000000..2757799b2f --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/effects/publish_result.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Terminal failure mapping and plaintext publication. + +use super::*; + +impl ThresholdPlaintextAggregator { + pub(in crate::actors::threshold_plaintext_aggregator) fn fail_decryption_round( + &mut self, + ec: EventContext, + ) -> Result<()> { + self.bus.publish( + E3Failed { + e3_id: self.e3_id.clone(), + failed_at_stage: E3Stage::CiphertextReady, + reason: FailureReason::DecryptionInvalidShares, + }, + ec, + )?; + + self.pending.honest_c6_proofs_for_agg = None; + self.pending.threshold_decryption_correlation = None; + self.pending.decryption_aggregation_correlation = None; + self.pending.c7_proofs_pending = None; + self.pending.decryption_aggregator_proofs = None; + + Ok(()) + } + + pub(in crate::actors::threshold_plaintext_aggregator) fn handle_compute_request_error( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + if msg.request().e3_id != self.e3_id { + return Ok(()); + } + + let threshold_decryption_failed = + self.pending.threshold_decryption_correlation.as_ref() == Some(msg.correlation_id()); + let decryption_aggregation_failed = + self.pending.decryption_aggregation_correlation.as_ref() == Some(msg.correlation_id()); + + if !threshold_decryption_failed && !decryption_aggregation_failed { + return Ok(()); + } + + // Surface the structured threshold-BFV failure when present so the implicated party and + // failure mode are visible in logs. Slashing/accusation stays driven by the C6 proof + // verification path; this is diagnostics only. + if let ComputeRequestErrorKind::TrBFV(trbfv_err) = msg.get_err() { + let failure = trbfv_err.failure(); + match &failure.threshold { + Some(threshold) => warn!( + e3_id = %self.e3_id, + kind = ?threshold.kind, + party_id = ?threshold.party_id, + "threshold decryption failed with structured error: {}", + threshold.message, + ), + None => warn!( + e3_id = %self.e3_id, + "threshold decryption failed: {}", + failure.message, + ), + } + } + + self.fail_decryption_round(ec) + } + + /// Publish `PlaintextAggregated` when both C7 proofs and decryption aggregation are complete. + pub(in crate::actors::threshold_plaintext_aggregator) fn try_publish_complete( + &mut self, + ) -> Result<()> { + let Some(c7_proofs) = self.pending.c7_proofs_pending.clone() else { + return Ok(()); + }; + let dec_ready = self.pending.decryption_aggregator_proofs.is_some() + && self.pending.decryption_aggregation_correlation.is_none(); + if !dec_ready { + return Ok(()); + } + + let state: GeneratingC7Proof = self + .state + .get() + .ok_or_else(|| anyhow!("Expected GeneratingC7Proof state"))? + .try_into()?; + + let ec = self + .pending + .last_ec + .clone() + .ok_or_else(|| anyhow!("No EventContext for publish"))?; + + info!("C7 + decryption_aggregator proofs ready — publishing PlaintextAggregated"); + + let decrypted_output = format_decrypted_plaintext(&state.plaintext); + + let decryption_aggregator_proofs = self + .pending + .decryption_aggregator_proofs + .clone() + .unwrap_or_default(); + // Keep c7_proofs for invariant check; they are subsumed by the decryption_aggregator proof. + let _ = c7_proofs; + let event = PlaintextAggregated { + decrypted_output, + e3_id: self.e3_id.clone(), + decryption_aggregator_proofs, + }; + + info!("Dispatching plaintext event {:?}", event); + self.bus.publish(event, ec.clone())?; + + self.state.try_mutate(&ec, |_| { + Ok(ThresholdPlaintextAggregatorState::Complete(Complete { + decrypted: state.plaintext, + shares: state.shares, + })) + })?; + + Ok(()) + } +} diff --git a/crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs b/crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs new file mode 100644 index 0000000000..ebb699433e --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/effects/verify_decryption_shares.rs @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Decryption-share collection and C6 verification. + +use super::*; + +impl ThresholdPlaintextAggregator { + pub fn add_share( + &mut self, + party_id: u64, + share: Vec, + signed_decryption_proofs: Vec, + ec: &EventContext, + ) -> Result<()> { + let required_shares = self.aggregated_committee_n(); + ensure!( + required_shares > 0, + "honest committee addresses must not be empty before collecting decryption shares" + ); + self.state.try_mutate(ec, |state| { + ThresholdPlaintextAggregation::add_share( + state, + party_id, + share.clone(), + signed_decryption_proofs.clone(), + required_shares, + ) + }) + } + + pub fn handle_member_expelled( + &mut self, + party_id: u64, + ec: &EventContext, + ) -> Result<()> { + let required_shares = self.aggregated_committee_n(); + self.state.try_mutate(ec, |state| { + ThresholdPlaintextAggregation::handle_member_expelled(state, party_id, required_shares) + }) + } + + /// Dispatch C6 proof verification through ShareVerificationActor. + pub fn dispatch_c6_verification( + &mut self, + c6_proofs: BTreeMap>, + ec: EventContext, + ) -> Result<()> { + let party_proofs = ThresholdPlaintextAggregation::plan_c6_dispatch(c6_proofs); + + self.bus.publish( + ShareVerificationDispatched { + e3_id: self.e3_id.clone(), + kind: VerificationKind::ThresholdDecryptionProofs, + share_proofs: party_proofs, + decryption_proofs: vec![], + pre_dishonest: BTreeSet::new(), + params_preset: self.params_preset, + committee_size: self.committee_size, + }, + ec, + )?; + Ok(()) + } + + /// Handle ShareVerificationComplete for C6: filter dishonest parties, transition to Computing. + pub fn handle_c6_verification_complete( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + + if msg.kind != VerificationKind::ThresholdDecryptionProofs { + return Ok(()); + } + + if msg.e3_id != self.e3_id { + return Ok(()); + } + + let state: VerifyingC6 = self + .state + .get() + .ok_or(anyhow!("Could not get state"))? + .try_into()?; + + let mut dishonest_parties = msg.dishonest_parties.clone(); + if !dishonest_parties.is_empty() { + warn!( + "C6 verification: {} dishonest parties filtered: {:?}", + dishonest_parties.len(), + dishonest_parties + ); + } + + // Filter shares to only honest parties + let mut honest_shares: Vec<(u64, Vec)> = state + .shares + .iter() + .filter(|(id, _)| !dishonest_parties.contains(id)) + .map(|(id, s)| (*id, s.clone())) + .collect(); + + if honest_shares.len() <= state.threshold_m as usize { + warn!( + "Not enough honest shares after C6 verification: {} honest shares, {} required", + honest_shares.len(), + state.threshold_m + 1 + ); + return self.fail_decryption_round(ec); + } + + // Verify each honest party's raw decryption share matches the + // d_commitment attested by their verified C6 proof. Catches the attack + // where a node sends a valid C6 proof for share d_A but broadcasts + // different bytes d_B. + let share_mismatch_parties = + ThresholdPlaintextAggregation::verify_shares_match_c6_commitments( + self.params_preset, + &honest_shares, + &state.c6_proofs, + ); + if !share_mismatch_parties.is_empty() { + warn!( + "C6 share-commitment mismatch for {} parties: {:?} — excluding from aggregation", + share_mismatch_parties.len(), + share_mismatch_parties, + ); + + dishonest_parties.extend(&share_mismatch_parties); + honest_shares.retain(|(id, _)| !share_mismatch_parties.contains(id)); + if honest_shares.len() <= state.threshold_m as usize { + warn!( + "Not enough honest shares after d_commitment check: {} honest, {} required", + honest_shares.len(), + state.threshold_m + 1 + ); + return self.fail_decryption_round(ec); + } + } + + info!( + "C6 verification passed: {} honest parties, transitioning to Computing", + honest_shares.len(), + ); + + // Collect honest C6 inner proofs (from signed payloads) for DecryptionAggregation. + // BTreeMap iteration yields ascending party_id, matching the slot layout + // used by honest_shares above and enforced by decryption_aggregator.nr. + let honest_c6: Vec<(u64, Vec)> = state + .c6_proofs + .iter() + .filter(|(id, _)| !dishonest_parties.contains(id)) + .map(|(id, signed)| { + ( + *id, + signed.iter().map(|s| s.payload.proof.clone()).collect(), + ) + }) + .collect(); + + // Publish ComputeRequest before transitioning state so a publish + // failure leaves us in VerifyingC6 (retryable) rather than + // Computing (no retry path). + // TrBFV scheme size stays N (`threshold_n`); only the share roster is restricted to the + // H canonical honest parties in `PublicKeyAggregated` (see + // `node_owns_aggregated_pk_party_slot`). + let trbfv_config = + TrBFVConfig::new(state.params.clone(), state.threshold_n, state.threshold_m); + + let correlation_id = CorrelationId::new(); + let event = ComputeRequest::trbfv( + TrBFVRequest::CalculateThresholdDecryption(CalculateThresholdDecryptionRequest { + ciphertexts: state.ciphertext_output.clone(), + trbfv_config, + d_share_polys: honest_shares.clone(), + }), + correlation_id, + self.e3_id.clone(), + ); + self.bus.publish(event, ec.clone())?; + + self.pending.honest_c6_proofs_for_agg = Some(honest_c6); + self.pending.threshold_decryption_correlation = Some(correlation_id); + + self.state.try_mutate(&ec, |_| { + Ok(ThresholdPlaintextAggregatorState::Computing(Computing { + shares: honest_shares, + ciphertext_output: state.ciphertext_output, + threshold_m: state.threshold_m, + threshold_n: state.threshold_n, + params: state.params, + })) + })?; + + self.pending.last_ec = Some(ec.clone()); + + Ok(()) + } +} diff --git a/crates/aggregator/src/plaintext_aggregation/handlers.rs b/crates/aggregator/src/plaintext_aggregation/handlers.rs new file mode 100644 index 0000000000..284c09c8f8 --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/handlers.rs @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Actix lifecycle, timeout ownership, and message routing. + +use super::*; + +impl Actor for ThresholdPlaintextAggregator { + type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + // Bound the decryption-share collection phase so a missing honest member cannot stall the + // round indefinitely. On expiry the round is failed loudly (see the timeout handler). + let timeout = decryption_collection_timeout(); + info!( + e3_id = %self.e3_id, + ?timeout, + "ThresholdPlaintextAggregator started; scheduling decryption-share collection timeout" + ); + self.pending.timeout_handle = Some(ctx.notify_later(DecryptionCollectionTimeout, timeout)); + } +} + +impl Handler for ThresholdPlaintextAggregator { + type Result = (); + fn handle(&mut self, _: DecryptionCollectionTimeout, ctx: &mut Self::Context) -> Self::Result { + self.pending.timeout_handle = None; + + // Only fail while still collecting shares; once we have transitioned past `Collecting` + // (VerifyingC6/Computing/…) the round is progressing and the timer is a no-op. + let Some(ThresholdPlaintextAggregatorState::Collecting(collecting)) = self.state.get() + else { + debug!( + e3_id = %self.e3_id, + "Decryption-share collection timeout fired but round already progressed past collection; ignoring" + ); + return; + }; + + let collected = collecting.shares.len(); + let required = self.aggregated_committee_n(); + warn!( + e3_id = %self.e3_id, + collected, + required, + "Decryption-share collection timed out with {collected}/{required} honest shares; failing E3 round (DecryptionTimeout)" + ); + + let Some(ec) = self.pending.timeout_ec.clone() else { + warn!( + e3_id = %self.e3_id, + "No event context captured for decryption timeout; cannot emit E3Failed. Stopping aggregator." + ); + ctx.stop(); + return; + }; + + if let Err(e) = self.bus.publish( + E3Failed { + e3_id: self.e3_id.clone(), + failed_at_stage: E3Stage::CiphertextReady, + reason: FailureReason::DecryptionTimeout, + }, + ec, + ) { + warn!( + e3_id = %self.e3_id, + error = %e, + "Failed to publish E3Failed on decryption-share collection timeout" + ); + } + + ctx.stop(); + } +} + +impl Handler for ThresholdPlaintextAggregator { + type Result = (); + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + match msg { + InterfoldEventData::DecryptionshareCreated(data) => { + ctx.notify(TypedEvent::new(data, ec)) + } + InterfoldEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), + InterfoldEventData::ComputeResponse(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ComputeRequestError(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::CommitteeMemberExpelled(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ShareVerificationComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::AggregationProofSigned(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + _ => (), + } + } +} + +impl Handler> for ThresholdPlaintextAggregator { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PublickeyAggregation, + &self.bus.with_ec(msg.get_ctx()), + || { + let Some(ThresholdPlaintextAggregatorState::Collecting(Collecting { .. })) = + self.state.get() + else { + debug!(state=?self.state, "Aggregator has been closed for collecting so ignoring this event."); + return Ok(()); + }; + let node = msg.node.clone(); + let e3_id = msg.e3_id.clone(); + let request = E3CommitteeContainsRequest::new(e3_id, node, msg, ctx.address()); + self.sortition.try_send(request)?; + Ok(()) + }, + ) + } +} + +impl Handler>> + for ThresholdPlaintextAggregator +{ + type Result = (); + fn handle( + &mut self, + msg: E3CommitteeContainsResponse>, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PublickeyAggregation, + &self.bus.with_ec(msg.get_ctx()), + || { + let e3_id = &msg.e3_id; + if *e3_id != self.e3_id { + bail!("Wrong e3_id sent to aggregator. This should not happen.") + }; + + if !msg.is_found_in_committee() { + trace!("Node {} not found in finalized committee", &msg.node); + return Ok(()); + }; + if !self.node_owns_aggregated_pk_party_slot(&msg.node, msg.party_id) { + trace!( + "Node {} does not own honest party slot {} — ignoring decryption share", + &msg.node, + msg.party_id + ); + return Ok(()); + } + + // Trust the party_id from the event - it's based on CommitteeFinalized order + // which is the authoritative source of truth for party IDs + let ( + DecryptionshareCreated { + party_id, + decryption_share, + signed_decryption_proofs, + .. + }, + ec, + ) = msg.into_inner().into_components(); + + // Capture the latest context so a subsequent collection timeout can emit + // `E3Failed` with a sensible causal parent. + self.pending.timeout_ec = Some(ec.clone()); + self.add_share(party_id, decryption_share, signed_decryption_proofs, &ec)?; + + // If we transitioned to VerifyingC6, dispatch C6 verification + // using the proofs persisted in state + if let Some(ThresholdPlaintextAggregatorState::VerifyingC6(ref state)) = + self.state.get() + { + self.dispatch_c6_verification(state.c6_proofs.clone(), ec)?; + } + + Ok(()) + }, + ) + } +} + +impl Handler> for ThresholdPlaintextAggregator { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PlaintextAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_compute_response(msg, ctx), + ) + } +} + +impl Handler> for ThresholdPlaintextAggregator { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PlaintextAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_compute_request_error(msg), + ) + } +} + +impl Handler> for ThresholdPlaintextAggregator { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PlaintextAggregation, + &self.bus.with_ec(msg.get_ctx()), + || { + let (msg, ec) = msg.into_components(); + let Some(party_id) = msg.party_id else { + return Ok(()); + }; + + self.handle_member_expelled(party_id, &ec)?; + + if let Some(ThresholdPlaintextAggregatorState::VerifyingC6(ref state)) = + self.state.get() + { + self.dispatch_c6_verification(state.c6_proofs.clone(), ec)?; + } + + Ok(()) + }, + ) + } +} + +impl Handler> for ThresholdPlaintextAggregator { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PlaintextAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_c6_verification_complete(msg), + ) + } +} + +impl Handler> for ThresholdPlaintextAggregator { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PlaintextAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_aggregation_proof_signed(msg, ctx), + ) + } +} + +impl Handler for ThresholdPlaintextAggregator { + type Result = (); + fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { + ctx.stop() + } +} diff --git a/crates/aggregator/src/plaintext_aggregation/intents.rs b/crates/aggregator/src/plaintext_aggregation/intents.rs new file mode 100644 index 0000000000..5059aba178 --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/intents.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Deterministic plaintext formatting and C7 aggregation job planning. + +use super::*; + +/// Pad/truncate each decrypted plaintext limb to the fixed `MAX_MSG_NON_ZERO_COEFFS * 8`. +pub(crate) fn format_decrypted_plaintext(plaintext: &[ArcBytes]) -> Vec { + let len = MAX_MSG_NON_ZERO_COEFFS * 8; + plaintext + .iter() + .map(|pt| { + let mut bytes = pt.extract_bytes(); + if bytes.len() >= len { + bytes.truncate(len); + } else { + bytes.resize(len, 0); + } + ArcBytes::from_bytes(&bytes) + }) + .collect() +} + +/// Bind each C7 (per-ciphertext) proof to the first `c6_total_slots` honest C6 inner +/// proofs for that ciphertext, producing the per-ciphertext decryption-aggregation jobs. +/// Returns `None` when an expected C6 inner proof is missing for some ciphertext index +/// (the actor then fails the decryption round). +pub(crate) fn build_decryption_aggregation_jobs( + c7_proofs: &[Proof], + honest_c6: &[(u64, Vec)], + c6_total_slots: usize, +) -> Option> { + let mut jobs = Vec::with_capacity(c7_proofs.len()); + for (ct_idx, c7_proof) in c7_proofs.iter().enumerate() { + let mut c6_inner_proofs = Vec::with_capacity(c6_total_slots); + let c6_slot_indices: Vec = (0..c6_total_slots as u32).collect(); + for (_, wps) in honest_c6.iter().take(c6_total_slots) { + let Some(p) = wps.get(ct_idx) else { + warn!("C6 inner proof missing for party at ct index {}", ct_idx); + return None; + }; + c6_inner_proofs.push(p.clone()); + } + jobs.push(DecryptionAggregationJobRequest { + c6_inner_proofs, + c6_slot_indices, + c7_proof: c7_proof.clone(), + }); + } + Some(jobs) +} diff --git a/crates/aggregator/src/plaintext_aggregation/state.rs b/crates/aggregator/src/plaintext_aggregation/state.rs new file mode 100644 index 0000000000..826265bb2d --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/state.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Persisted threshold-plaintext state schema. + +use super::*; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Collecting { + pub(crate) threshold_m: u64, + pub(crate) threshold_n: u64, + pub(crate) shares: BTreeMap>, + /// Signed raw C6 proofs for ShareVerification. + pub(crate) c6_proofs: BTreeMap>, + pub(crate) seed: Seed, + pub(crate) ciphertext_output: Vec, + pub(crate) params: ArcBytes, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct VerifyingC6 { + pub(crate) threshold_m: u64, + pub(crate) threshold_n: u64, + pub(crate) shares: BTreeMap>, + pub(crate) c6_proofs: BTreeMap>, + pub(crate) ciphertext_output: Vec, + pub(crate) params: ArcBytes, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Computing { + pub(crate) threshold_m: u64, + pub(crate) threshold_n: u64, + pub(crate) shares: Vec<(u64, Vec)>, + pub(crate) ciphertext_output: Vec, + pub(crate) params: ArcBytes, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct GeneratingC7Proof { + pub(crate) threshold_m: u64, + pub(crate) threshold_n: u64, + pub(crate) shares: Vec<(u64, Vec)>, + pub(crate) plaintext: Vec, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct Complete { + pub(crate) decrypted: Vec, + pub(crate) shares: Vec<(u64, Vec)>, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum ThresholdPlaintextAggregatorState { + Collecting(Collecting), + VerifyingC6(VerifyingC6), + Computing(Computing), + GeneratingC7Proof(GeneratingC7Proof), + Complete(Complete), +} + +impl TryFrom for Collecting { + type Error = anyhow::Error; + fn try_from( + value: ThresholdPlaintextAggregatorState, + ) -> std::result::Result { + match value { + ThresholdPlaintextAggregatorState::Collecting(s) => Ok(s), + _ => bail!("PlaintextState was expected to be Collecting but it was not."), + } + } +} + +impl TryFrom for VerifyingC6 { + type Error = anyhow::Error; + fn try_from( + value: ThresholdPlaintextAggregatorState, + ) -> std::result::Result { + match value { + ThresholdPlaintextAggregatorState::VerifyingC6(s) => Ok(s), + _ => bail!("Inconsistent state: expected VerifyingC6"), + } + } +} + +impl TryFrom for Computing { + type Error = anyhow::Error; + fn try_from( + value: ThresholdPlaintextAggregatorState, + ) -> std::result::Result { + match value { + ThresholdPlaintextAggregatorState::Computing(s) => Ok(s), + _ => bail!("Inconsistent state: expected Computing"), + } + } +} + +impl TryFrom for GeneratingC7Proof { + type Error = anyhow::Error; + fn try_from( + value: ThresholdPlaintextAggregatorState, + ) -> std::result::Result { + match value { + ThresholdPlaintextAggregatorState::GeneratingC7Proof(s) => Ok(s), + _ => bail!("Inconsistent state: expected GeneratingC7Proof"), + } + } +} + +impl TryFrom for Complete { + type Error = anyhow::Error; + fn try_from( + value: ThresholdPlaintextAggregatorState, + ) -> std::result::Result { + match value { + ThresholdPlaintextAggregatorState::Complete(s) => Ok(s), + _ => bail!("Inconsistent state: expected Complete"), + } + } +} + +impl ThresholdPlaintextAggregatorState { + pub fn init( + threshold_m: u64, + threshold_n: u64, + seed: Seed, + ciphertext_output: Vec, + params: ArcBytes, + ) -> Self { + ThresholdPlaintextAggregatorState::Collecting(Collecting { + threshold_m, + threshold_n, + shares: BTreeMap::new(), + c6_proofs: BTreeMap::new(), + seed, + ciphertext_output, + params, + }) + } +} diff --git a/crates/aggregator/src/plaintext_aggregation/tests/completion.rs b/crates/aggregator/src/plaintext_aggregation/tests/completion.rs new file mode 100644 index 0000000000..8b0bb3475a --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/tests/completion.rs @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn missing_c6_inner_proofs_emit_e3_failed() -> Result<()> { + let (mut aggregator, history, e3_id) = + build_plaintext_aggregator(generating_c7_state(), true).await?; + aggregator.pending.c7_proofs_pending = Some(vec![dummy_proof(CircuitName::PkAggregation)]); + aggregator.pending.honest_c6_proofs_for_agg = Some(vec![ + (0, vec![]), + (1, vec![dummy_proof(CircuitName::ThresholdShareDecryption)]), + ]); + + let ec = test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::None, + reason: FailureReason::None, + }); + aggregator.dispatch_decryption_aggregation(&ec)?; + + let event = next_event(&history).await?; + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == e3_id + && data.failed_at_stage == E3Stage::CiphertextReady + && data.reason == FailureReason::DecryptionInvalidShares + )); + assert!(aggregator.pending.honest_c6_proofs_for_agg.is_none()); + assert!(aggregator + .pending + .decryption_aggregation_correlation + .is_none()); + assert!(aggregator.pending.c7_proofs_pending.is_none()); + assert!(aggregator.pending.decryption_aggregator_proofs.is_none()); + + Ok(()) +} + +#[actix::test] +async fn proof_aggregation_disabled_marks_decryption_aggregator_ready() -> Result<()> { + let (mut aggregator, _history, _e3_id) = + build_plaintext_aggregator(generating_c7_state(), false).await?; + aggregator.pending.c7_proofs_pending = Some(vec![dummy_proof(CircuitName::PkAggregation)]); + let ec = test_ctx(E3Failed { + e3_id: aggregator.e3_id.clone(), + failed_at_stage: E3Stage::None, + reason: FailureReason::None, + }); + + aggregator.dispatch_decryption_aggregation(&ec)?; + assert!(aggregator + .pending + .decryption_aggregator_proofs + .as_ref() + .is_some_and(|p| p.is_empty())); + assert!(aggregator + .pending + .decryption_aggregation_correlation + .is_none()); + + Ok(()) +} diff --git a/crates/aggregator/src/plaintext_aggregation/tests/failures.rs b/crates/aggregator/src/plaintext_aggregation/tests/failures.rs new file mode 100644 index 0000000000..de7ef61d08 --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/tests/failures.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn decryption_collection_timeout_fails_round_while_collecting() -> Result<()> { + let (mut aggregator, history, e3_id) = + build_plaintext_aggregator(collecting_state(), true).await?; + // A captured context is required so the timeout can emit E3Failed with a causal parent. + aggregator.pending.timeout_ec = Some(test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CiphertextReady, + reason: FailureReason::None, + })); + let addr = aggregator.start(); + + addr.send(DecryptionCollectionTimeout).await?; + + let event = next_event(&history).await?; + assert!( + matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.reason == FailureReason::DecryptionTimeout + ), + "expected E3Failed with DecryptionTimeout when collection window elapses" + ); + Ok(()) +} + +#[actix::test] +async fn threshold_decryption_compute_error_emits_e3_failed() -> Result<()> { + let correlation_id = CorrelationId::new(); + let (mut aggregator, history, e3_id) = + build_plaintext_aggregator(computing_state(), true).await?; + aggregator.pending.threshold_decryption_correlation = Some(correlation_id); + + let request = ComputeRequest::trbfv( + TrBFVRequest::CalculateThresholdDecryption(CalculateThresholdDecryptionRequest { + ciphertexts: vec![ArcBytes::from_bytes(&[8])], + trbfv_config: TrBFVConfig::new(test_params(), 2, 1), + d_share_polys: vec![(0, vec![ArcBytes::from_bytes(&[7])])], + }), + correlation_id, + e3_id.clone(), + ); + + aggregator.handle_compute_request_error(TypedEvent::new( + ComputeRequestError::new( + ComputeRequestErrorKind::TrBFV(e3_trbfv::TrBFVError::CalculateThresholdDecryption( + "boom".into(), + )), + request, + ), + test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::None, + reason: FailureReason::None, + }), + ))?; + + let event = next_event(&history).await?; + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == e3_id + && data.failed_at_stage == E3Stage::CiphertextReady + && data.reason == FailureReason::DecryptionInvalidShares + )); + assert!(aggregator + .pending + .threshold_decryption_correlation + .is_none()); + + Ok(()) +} + +#[actix::test] +async fn insufficient_honest_c6_shares_emit_e3_failed() -> Result<()> { + let (mut aggregator, history, e3_id) = + build_plaintext_aggregator(verifying_c6_state(), true).await?; + + aggregator.handle_c6_verification_complete(TypedEvent::new( + ShareVerificationComplete { + e3_id: e3_id.clone(), + kind: VerificationKind::ThresholdDecryptionProofs, + dishonest_parties: BTreeSet::from([1]), + }, + test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::None, + reason: FailureReason::None, + }), + ))?; + + let event = next_event(&history).await?; + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == e3_id + && data.failed_at_stage == E3Stage::CiphertextReady + && data.reason == FailureReason::DecryptionInvalidShares + )); + + Ok(()) +} + +#[actix::test] +async fn decryption_aggregation_compute_error_emits_e3_failed() -> Result<()> { + let correlation_id = CorrelationId::new(); + let (mut aggregator, history, e3_id) = + build_plaintext_aggregator(generating_c7_state(), true).await?; + aggregator.pending.c7_proofs_pending = Some(vec![dummy_proof(CircuitName::PkAggregation)]); + aggregator.pending.honest_c6_proofs_for_agg = Some(vec![( + 0, + vec![dummy_proof(CircuitName::ThresholdShareDecryption)], + )]); + aggregator.pending.decryption_aggregation_correlation = Some(correlation_id); + aggregator.pending.last_ec = Some(test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::None, + reason: FailureReason::None, + })); + + let request = ComputeRequest::zk( + ZkRequest::DecryptionAggregation(DecryptionAggregationRequest { + c6_total_slots: 1, + jobs: Vec::new(), + committee_addresses: vec![test_committee_address()], + params_preset: BfvPreset::InsecureThreshold512, + committee_size: CiphernodesCommitteeSize::Minimum, + }), + correlation_id, + e3_id.clone(), + ); + + aggregator.handle_compute_request_error(TypedEvent::new( + ComputeRequestError::new( + ComputeRequestErrorKind::Zk(ZkError::ProofGenerationFailed("boom".to_string())), + request, + ), + test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::None, + reason: FailureReason::None, + }), + ))?; + + let event = next_event(&history).await?; + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == e3_id + && data.failed_at_stage == E3Stage::CiphertextReady + && data.reason == FailureReason::DecryptionInvalidShares + )); + assert!(aggregator + .pending + .decryption_aggregation_correlation + .is_none()); + assert!(aggregator.pending.c7_proofs_pending.is_none()); + + Ok(()) +} diff --git a/crates/aggregator/src/plaintext_aggregation/tests/mod.rs b/crates/aggregator/src/plaintext_aggregation/tests/mod.rs new file mode 100644 index 0000000000..5657bd018b --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/tests/mod.rs @@ -0,0 +1,181 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use e3_data::{AutoPersist, DataStore, InMemStore, PersistableData, Repository}; +use e3_events::{ + CircuitName, Committee, ComputeRequestErrorKind, HistoryCollector, Seed, TakeEvents, + Unsequenced, ZkError, +}; +use e3_fhe_params::{encode_bfv_params, BfvParamSet, DEFAULT_BFV_PRESET}; +use e3_sortition::{ + CiphernodeSelector, CiphernodeSelectorState, NodeStateStore, SortitionBackend, SortitionParams, +}; +use e3_test_helpers::get_common_setup; +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +fn test_ctx(data: impl Into) -> EventContext { + EventContext::::from(data.into()).sequence(0) +} + +fn test_persistable(value: T) -> Persistable { + let repo = Repository::::new(DataStore::from_in_mem(&InMemStore::new(false).start())); + repo.to_connector().send(Some(value)) +} + +fn test_params() -> ArcBytes { + ArcBytes::from_bytes(&encode_bfv_params( + &BfvParamSet::from(DEFAULT_BFV_PRESET).build_arc(), + )) +} + +fn dummy_proof(circuit: CircuitName) -> Proof { + Proof::new( + circuit, + ArcBytes::from_bytes(&[1]), + ArcBytes::from_bytes(&[2]), + ) +} + +#[test] +fn decryption_sender_must_own_an_honest_party_slot() { + let first = Address::repeat_byte(0x11); + let second = Address::repeat_byte(0x22); + let committee = [first, second]; + let honest = [first]; + + assert!(node_owns_committee_party_slot( + &committee, + &honest, + &first.to_string(), + 0 + )); + assert!(!node_owns_committee_party_slot( + &committee, + &honest, + &first.to_string(), + 1 + )); + assert!(!node_owns_committee_party_slot( + &committee, + &honest, + &second.to_string(), + 1 + )); + assert!(!node_owns_committee_party_slot( + &committee, + &honest, + &first.to_string(), + 2 + )); +} + +fn computing_state() -> ThresholdPlaintextAggregatorState { + ThresholdPlaintextAggregatorState::Computing(Computing { + threshold_m: 1, + threshold_n: 2, + shares: vec![(0, vec![ArcBytes::from_bytes(&[7])])], + ciphertext_output: vec![ArcBytes::from_bytes(&[8])], + params: test_params(), + }) +} + +fn verifying_c6_state() -> ThresholdPlaintextAggregatorState { + ThresholdPlaintextAggregatorState::VerifyingC6(VerifyingC6 { + threshold_m: 1, + threshold_n: 2, + shares: BTreeMap::from([ + (0, vec![ArcBytes::from_bytes(&[7])]), + (1, vec![ArcBytes::from_bytes(&[8])]), + ]), + c6_proofs: BTreeMap::new(), + ciphertext_output: vec![ArcBytes::from_bytes(&[9])], + params: test_params(), + }) +} + +fn generating_c7_state() -> ThresholdPlaintextAggregatorState { + ThresholdPlaintextAggregatorState::GeneratingC7Proof(GeneratingC7Proof { + threshold_m: 1, + threshold_n: 2, + shares: vec![(0, vec![ArcBytes::from_bytes(&[7])])], + plaintext: vec![ArcBytes::from_bytes(&[9])], + }) +} + +fn collecting_state() -> ThresholdPlaintextAggregatorState { + ThresholdPlaintextAggregatorState::Collecting(Collecting { + threshold_m: 1, + threshold_n: 2, + shares: BTreeMap::new(), + c6_proofs: BTreeMap::new(), + seed: Seed([0u8; 32]), + ciphertext_output: vec![ArcBytes::from_bytes(&[9])], + params: test_params(), + }) +} + +fn start_sortition(bus: &BusHandle) -> Addr { + let selector = CiphernodeSelector::new( + bus, + test_persistable(CiphernodeSelectorState::default()), + "node-1", + ) + .start(); + + Sortition::new(SortitionParams { + bus: bus.clone(), + backends: test_persistable(HashMap::::new()), + node_state: test_persistable(HashMap::::new()), + finalized_committees: test_persistable(HashMap::::new()), + ciphernode_selector: selector, + address: "node-1".to_string(), + }) + .start() +} + +fn test_committee_address() -> Address { + "0x0000000000000000000000000000000000000001" + .parse() + .expect("test address") +} + +async fn build_plaintext_aggregator( + initial_state: ThresholdPlaintextAggregatorState, + proof_aggregation_enabled: bool, +) -> Result<( + ThresholdPlaintextAggregator, + Addr>, + E3id, +)> { + let (bus, _rng, _seed, _params, _crp, _errors, history) = + get_common_setup(Some(BfvPreset::InsecureThreshold512.into()))?; + let e3_id = E3id::new("42", 1); + let aggregator = ThresholdPlaintextAggregator::new( + ThresholdPlaintextAggregatorParams { + bus: bus.clone(), + sortition: start_sortition(&bus), + e3_id: e3_id.clone(), + params_preset: BfvPreset::InsecureThreshold512, + committee_size: CiphernodesCommitteeSize::Minimum, + proof_aggregation_enabled, + committee_addresses: vec![test_committee_address()], + honest_committee_addresses: vec![test_committee_address()], + }, + test_persistable(initial_state), + ); + + Ok((aggregator, history, e3_id)) +} + +async fn next_event(history: &Addr>) -> Result { + let mut result = history.send(TakeEvents::::new(1)).await?; + assert!(!result.timed_out, "timed out waiting for an event"); + Ok(result.events.pop().expect("expected one event")) +} + +mod completion; +mod failures; diff --git a/crates/aggregator/src/plaintext_aggregation/transitions.rs b/crates/aggregator/src/plaintext_aggregation/transitions.rs new file mode 100644 index 0000000000..63e21cceb5 --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/transitions.rs @@ -0,0 +1,247 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Pure share collection, expulsion, and C6 verification transitions. + +use super::*; + +/// Plain, synchronous domain service for threshold-plaintext aggregation decisions. +pub(crate) struct ThresholdPlaintextAggregation; + +impl ThresholdPlaintextAggregation { + /// Add a decryption share to a `Collecting` state, returning the next state. Once all + /// `required_shares` honest-committee shares have arrived this transitions to `VerifyingC6`. + /// `required_shares` is the canonical honest-committee size `H` (computed by the actor). + pub(crate) fn add_share( + state: ThresholdPlaintextAggregatorState, + party_id: u64, + share: Vec, + signed_decryption_proofs: Vec, + required_shares: u64, + ) -> Result { + info!("Adding share for party_id={}", party_id); + let current: Collecting = state.try_into()?; + let expected_outputs = current.ciphertext_output.len(); + ensure!( + share.len() == expected_outputs, + "party {party_id} supplied {} decryption shares for {expected_outputs} ciphertext outputs", + share.len() + ); + ensure!( + signed_decryption_proofs.len() == expected_outputs, + "party {party_id} supplied {} C6 proofs for {expected_outputs} ciphertext outputs", + signed_decryption_proofs.len() + ); + let ciphertext_output = current.ciphertext_output; + let threshold_m = current.threshold_m; + let threshold_n = current.threshold_n; + let params = current.params.clone(); + let mut shares = current.shares; + let mut c6_proofs = current.c6_proofs; + + info!("pushing to share collection {} {:?}", party_id, share); + shares.insert(party_id, share); + c6_proofs.insert(party_id, signed_decryption_proofs); + + if (shares.len() as u64) < required_shares { + return Ok(ThresholdPlaintextAggregatorState::Collecting(Collecting { + params, + threshold_n, + threshold_m, + ciphertext_output, + shares, + c6_proofs, + seed: current.seed, + })); + } + + info!( + "Changing state to VerifyingC6 because received all {required_shares} honest-committee shares..." + ); + + Ok(ThresholdPlaintextAggregatorState::VerifyingC6( + VerifyingC6 { + shares, + c6_proofs, + ciphertext_output, + threshold_m, + threshold_n, + params, + }, + )) + } + + /// Apply a committee-member expulsion to a `Collecting` state, removing the party's share + /// and C6 proofs, and transitioning to `VerifyingC6` when enough shares remain. + pub(crate) fn handle_member_expelled( + state: ThresholdPlaintextAggregatorState, + party_id: u64, + required_shares: u64, + ) -> Result { + let ThresholdPlaintextAggregatorState::Collecting(current) = state else { + return Ok(state); + }; + + let mut shares = current.shares; + let mut c6_proofs = current.c6_proofs; + let threshold_n = current.threshold_n; + + shares.remove(&party_id); + c6_proofs.remove(&party_id); + + if required_shares < current.threshold_m { + warn!( + "ThresholdPlaintextAggregator: honest committee size H ({required_shares}) < threshold_m ({}) after expulsion", + current.threshold_m + ); + return Ok(ThresholdPlaintextAggregatorState::Collecting(Collecting { + threshold_m: current.threshold_m, + threshold_n, + shares, + c6_proofs, + seed: current.seed, + ciphertext_output: current.ciphertext_output, + params: current.params, + })); + } + + if (shares.len() as u64) < required_shares { + return Ok(ThresholdPlaintextAggregatorState::Collecting(Collecting { + threshold_m: current.threshold_m, + threshold_n, + shares, + c6_proofs, + seed: current.seed, + ciphertext_output: current.ciphertext_output, + params: current.params, + })); + } + + Ok(ThresholdPlaintextAggregatorState::VerifyingC6( + VerifyingC6 { + threshold_m: current.threshold_m, + threshold_n, + shares, + c6_proofs, + ciphertext_output: current.ciphertext_output, + params: current.params, + }, + )) + } + + /// Build the per-party C6 proof bundles dispatched to ShareVerification. + pub(crate) fn plan_c6_dispatch( + c6_proofs: BTreeMap>, + ) -> Vec { + c6_proofs + .into_iter() + .map(|(party_id, signed_proofs)| PartyProofsToVerify { + sender_party_id: party_id, + signed_proofs, + }) + .collect() + } + + /// Verify that each honest party's raw decryption share bytes match the + /// `d_commitment` output in their verified C6 proof. Returns party IDs + /// that failed the check. + /// + /// Catches the attack where a node sends a valid C6 proof for share `d_A` but + /// broadcasts different bytes `d_B`. + pub(crate) fn verify_shares_match_c6_commitments( + params_preset: BfvPreset, + honest_shares: &[(u64, Vec)], + c6_proofs: &BTreeMap>, + ) -> BTreeSet { + let mut mismatched = BTreeSet::new(); + + let Ok((threshold_params, _)) = e3_fhe_params::build_pair_for_preset(params_preset) else { + warn!("Could not build BFV params for d_commitment check — skipping"); + return mismatched; + }; + + // Reuse the same Bounds/Bits computation that C6 codegen uses, + // so d_native_bit stays in sync if the formula ever changes. + let Ok(bounds) = C6Bounds::compute(params_preset, &()) else { + warn!("Could not compute bounds for d_commitment check — skipping"); + return mismatched; + }; + let Ok(bits) = C6Bits::compute(params_preset, &bounds) else { + warn!("Could not compute bits for d_commitment check — skipping"); + return mismatched; + }; + let d_native_bit = bits.d_native_bit; + + let max_k = MAX_MSG_NON_ZERO_COEFFS; + let c6_output_layout = CircuitName::ThresholdShareDecryption.output_layout(); + + for (party_id, shares) in honest_shares { + let Some(proofs) = c6_proofs.get(party_id) else { + warn!( + "No C6 proofs for party {} — marking as mismatched", + party_id + ); + mismatched.insert(*party_id); + continue; + }; + let Some(first_proof) = proofs.first() else { + warn!( + "Empty C6 proof list for party {} — marking as mismatched", + party_id + ); + mismatched.insert(*party_id); + continue; + }; + let Some(c6_d_bytes) = c6_output_layout + .extract_field(&first_proof.payload.proof.public_signals, "d_commitment") + else { + warn!( + "Could not extract d_commitment from C6 proof for party {} — marking as mismatched", + party_id + ); + mismatched.insert(*party_id); + continue; + }; + + let Some(share_bytes) = shares.first() else { + warn!( + "No share bytes for party {} — marking as mismatched", + party_id + ); + mismatched.insert(*party_id); + continue; + }; + let Ok(poly) = + e3_trbfv::helpers::try_poly_pb_from_bytes(share_bytes, &threshold_params) + else { + warn!( + "Could not deserialize share for party {} — marking as mismatched", + party_id + ); + mismatched.insert(*party_id); + continue; + }; + let crt = e3_polynomial::CrtPolynomial::from_fhe_polynomial(&poly); + + // C6 public `d_commitment` hashes native truncated limbs (same layout as C7), not + // reversed+centered witness `d`. + let computed = compute_threshold_decryption_share_commitment(&crt, d_native_bit, max_k); + + // Convert to big-endian 32-byte padded format matching + // Barretenberg's public_signals encoding. + let (_, be_bytes) = computed.to_bytes_be(); + let mut computed_padded = [0u8; 32]; + let start = 32usize.saturating_sub(be_bytes.len()); + computed_padded[start..].copy_from_slice(&be_bytes[..be_bytes.len().min(32)]); + + if computed_padded != c6_d_bytes { + warn!( + "d_commitment mismatch for party {}: raw share commitment differs from C6 proof output", + party_id + ); + mismatched.insert(*party_id); + } + } + + mismatched + } +} diff --git a/crates/aggregator/src/plaintext_aggregation/workflow.rs b/crates/aggregator/src/plaintext_aggregation/workflow.rs new file mode 100644 index 0000000000..0c4fa910a8 --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/workflow.rs @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Deterministic threshold-plaintext (decryption) aggregation workflow. +//! +//! This module holds the [`ThresholdPlaintextAggregatorState`] state machine plus the pure +//! transition/decision functions used by the `ThresholdPlaintextAggregator` actor. Nothing +//! here touches actix, `Persistable`, or the event bus: the actor feeds inputs in, gets a +//! next-state or a decision back, and performs the persistence/publish/dispatch side effects +//! itself. + +use std::collections::{BTreeMap, BTreeSet}; + +use anyhow::{bail, ensure, Result}; +use e3_events::CircuitName; +use e3_events::{ + DecryptionAggregationJobRequest, PartyProofsToVerify, Proof, Seed, SignedProofPayload, +}; +use e3_fhe_params::BfvPreset; +use e3_utils::utility_types::ArcBytes; +use e3_zk_helpers::circuits::commitments::compute_threshold_decryption_share_commitment; +use e3_zk_helpers::circuits::threshold::decrypted_shares_aggregation::MAX_MSG_NON_ZERO_COEFFS; +use e3_zk_helpers::threshold::share_decryption::{Bits as C6Bits, Bounds as C6Bounds}; +use e3_zk_helpers::Computation; +use tracing::{info, warn}; + +#[path = "intents.rs"] +mod intents; +#[path = "state.rs"] +mod state; +#[path = "transitions.rs"] +mod transitions; + +pub(crate) use intents::{build_decryption_aggregation_jobs, format_decrypted_plaintext}; +pub use state::{ + Collecting, Complete, Computing, GeneratingC7Proof, ThresholdPlaintextAggregatorState, + VerifyingC6, +}; +pub(crate) use transitions::ThresholdPlaintextAggregation; + +#[cfg(test)] +#[path = "workflow_tests.rs"] +mod tests; diff --git a/crates/aggregator/src/plaintext_aggregation/workflow_tests.rs b/crates/aggregator/src/plaintext_aggregation/workflow_tests.rs new file mode 100644 index 0000000000..5cb2ce4495 --- /dev/null +++ b/crates/aggregator/src/plaintext_aggregation/workflow_tests.rs @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +fn ab(b: u8) -> ArcBytes { + ArcBytes::from_bytes(&[b]) +} + +fn c6_proof(marker: u8) -> SignedProofPayload { + SignedProofPayload { + payload: e3_events::ProofPayload { + e3_id: e3_events::E3id::new("1", 1), + proof_type: e3_events::ProofType::C6ThresholdShareDecryption, + proof: Proof::new( + CircuitName::ThresholdShareDecryption, + ab(marker), + ab(marker), + ), + }, + signature: ab(marker), + } +} + +fn collecting(threshold_m: u64, threshold_n: u64) -> ThresholdPlaintextAggregatorState { + ThresholdPlaintextAggregatorState::init( + threshold_m, + threshold_n, + Seed([0u8; 32]), + vec![ab(1)], + ab(2), + ) +} + +#[test] +fn add_share_below_required_stays_collecting() { + let state = collecting(1, 3); + let next = + ThresholdPlaintextAggregation::add_share(state, 0, vec![ab(10)], vec![c6_proof(10)], 3) + .unwrap(); + match next { + ThresholdPlaintextAggregatorState::Collecting(c) => { + assert_eq!(c.shares.len(), 1); + assert!(c.shares.contains_key(&0)); + } + _ => panic!("expected Collecting"), + } +} + +#[test] +fn add_share_reaching_required_transitions_to_verifying_c6() { + let mut state = collecting(1, 3); + for pid in 0..3u64 { + state = ThresholdPlaintextAggregation::add_share( + state, + pid, + vec![ab(pid as u8)], + vec![c6_proof(pid as u8)], + 3, + ) + .unwrap(); + } + match state { + ThresholdPlaintextAggregatorState::VerifyingC6(v) => { + assert_eq!(v.shares.len(), 3); + } + _ => panic!("expected VerifyingC6"), + } +} + +#[test] +fn add_share_wrong_state_errors() { + let state = ThresholdPlaintextAggregatorState::VerifyingC6(VerifyingC6 { + threshold_m: 1, + threshold_n: 3, + shares: BTreeMap::new(), + c6_proofs: BTreeMap::new(), + ciphertext_output: vec![ab(1)], + params: ab(2), + }); + let res = ThresholdPlaintextAggregation::add_share(state, 0, vec![ab(0)], vec![], 3); + assert!(res.is_err()); +} + +#[test] +fn handle_member_expelled_removes_share_and_stays_collecting() { + let mut state = collecting(1, 3); + for pid in 0..2u64 { + state = ThresholdPlaintextAggregation::add_share( + state, + pid, + vec![ab(pid as u8)], + vec![c6_proof(pid as u8)], + 3, + ) + .unwrap(); + } + // required_shares stays 3; remove party 0 -> 1 share left -> Collecting + let next = ThresholdPlaintextAggregation::handle_member_expelled(state, 0, 3).unwrap(); + match next { + ThresholdPlaintextAggregatorState::Collecting(c) => { + assert_eq!(c.shares.len(), 1); + assert!(!c.shares.contains_key(&0)); + } + _ => panic!("expected Collecting"), + } +} + +#[test] +fn handle_member_expelled_transitions_when_enough_remain() { + let mut state = collecting(1, 3); + for pid in 0..3u64 { + state = ThresholdPlaintextAggregation::add_share( + state, + pid, + vec![ab(pid as u8)], + vec![c6_proof(pid as u8)], + 3, + ) + .unwrap(); + } + // After 3 shares it's already VerifyingC6; rebuild a Collecting with 3 shares to + // exercise the expulsion->VerifyingC6 path with required_shares lowered to 2. + let state = ThresholdPlaintextAggregatorState::Collecting(Collecting { + threshold_m: 1, + threshold_n: 3, + shares: BTreeMap::from([(0, vec![ab(0)]), (1, vec![ab(1)]), (2, vec![ab(2)])]), + c6_proofs: BTreeMap::new(), + seed: Seed([0u8; 32]), + ciphertext_output: vec![ab(1)], + params: ab(2), + }); + let _ = state; + let state = ThresholdPlaintextAggregatorState::Collecting(Collecting { + threshold_m: 1, + threshold_n: 3, + shares: BTreeMap::from([(0, vec![ab(0)]), (1, vec![ab(1)]), (2, vec![ab(2)])]), + c6_proofs: BTreeMap::new(), + seed: Seed([0u8; 32]), + ciphertext_output: vec![ab(1)], + params: ab(2), + }); + // remove party 0 -> 2 shares remain, required_shares=2 -> VerifyingC6 + let next = ThresholdPlaintextAggregation::handle_member_expelled(state, 0, 2).unwrap(); + match next { + ThresholdPlaintextAggregatorState::VerifyingC6(v) => { + assert_eq!(v.shares.len(), 2); + } + _ => panic!("expected VerifyingC6"), + } +} + +#[test] +fn handle_member_expelled_wrong_state_is_noop() { + let state = ThresholdPlaintextAggregatorState::Complete(Complete { + decrypted: vec![ab(1)], + shares: vec![], + }); + let next = ThresholdPlaintextAggregation::handle_member_expelled(state, 0, 3).unwrap(); + assert!(matches!( + next, + ThresholdPlaintextAggregatorState::Complete(_) + )); +} + +#[test] +fn add_share_rejects_c6_share_or_proof_count_mismatch() { + let missing_share = + ThresholdPlaintextAggregation::add_share(collecting(1, 3), 0, vec![], vec![c6_proof(1)], 3) + .expect_err("one decryption share is required for one ciphertext"); + assert!(missing_share.to_string().contains("decryption shares")); + + let missing_proof = + ThresholdPlaintextAggregation::add_share(collecting(1, 3), 0, vec![ab(1)], vec![], 3) + .expect_err("one C6 proof is required for one ciphertext"); + assert!(missing_proof.to_string().contains("C6 proofs")); +} + +#[test] +fn plan_c6_dispatch_emits_party_proofs_in_party_order() { + let mut c6: BTreeMap> = BTreeMap::new(); + c6.insert(2, vec![]); + c6.insert(0, vec![]); + c6.insert(1, vec![]); + let plan = ThresholdPlaintextAggregation::plan_c6_dispatch(c6); + let ids: Vec = plan.iter().map(|p| p.sender_party_id).collect(); + assert_eq!(ids, vec![0, 1, 2]); +} diff --git a/crates/aggregator/src/public_key_aggregation/actor.rs b/crates/aggregator/src/public_key_aggregation/actor.rs new file mode 100644 index 0000000000..5b3de16c3b --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/actor.rs @@ -0,0 +1,86 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::domain::committee::committee_addresses_in_party_order; +use crate::workflow::publickey_aggregation::{ + check_c1_keyshare_commitments, extract_pk_commitment, verify_dkg_fold_attestation, C1Dispatch, + HonestSelection, PublicKeyAggregation, +}; +use actix::prelude::*; +use anyhow::Result; +use e3_data::Persistable; +use e3_events::{ + prelude::*, BusHandle, ComputeRequest, ComputeRequestError, ComputeResponse, + ComputeResponseKind, CorrelationId, DKGRecursiveAggregationComplete, Die, + DkgAggregationRequest, E3Failed, E3Stage, E3id, EventContext, FailureReason, InterfoldEvent, + InterfoldEventData, KeyshareCreated, NodesFoldStepRequest, OrderedSet, + PkAggregationProofPending, PkAggregationProofRequest, PkAggregationProofSigned, Proof, + ProofType, PublicKeyAggregated, Sequenced, ShareVerificationComplete, + ShareVerificationDispatched, SignedProofFailed, SignedProofPayload, TypedEvent, + VerificationKind, ZkRequest, ZkResponse, +}; +use e3_events::{trap, EType}; +use e3_fhe::{Fhe, GetAggregatePublicKey}; +use e3_fhe_params::BfvPreset; +use e3_utils::NotifySync; +use e3_utils::{ArcBytes, MAILBOX_LIMIT}; +use e3_zk_helpers::CiphernodesCommitteeSize; +use std::collections::HashMap; +use std::sync::Arc; +use tracing::{error, info, warn}; + +// Public-key aggregation state machine + pure transition logic now live in +// `crate::workflow::publickey_aggregation`; re-exported here to preserve the public path +// `e3_aggregator::publickey_aggregator::PublicKeyAggregatorState`. +pub use crate::workflow::publickey_aggregation::PublicKeyAggregatorState; + +pub struct PublicKeyAggregator { + fhe: Arc, + bus: BusHandle, + e3_id: E3id, + state: Persistable, + params_preset: BfvPreset, + committee_size: CiphernodesCommitteeSize, + /// DKG recursive aggregation events received before entering GeneratingC5Proof. + early_dkg_proofs: Vec>, +} + +pub struct PublicKeyAggregatorParams { + pub fhe: Arc, + pub bus: BusHandle, + pub e3_id: E3id, + pub params_preset: BfvPreset, + pub committee_size: CiphernodesCommitteeSize, +} + +/// Aggregate PublicKey for a committee of nodes. This actor listens for KeyshareCreated events +/// around a particular e3_id, verifies C1 proofs, aggregates the public key, generates a C5 +/// proof of correct aggregation, and broadcasts a PublicKeyAggregated event on the event bus. +impl PublicKeyAggregator { + pub fn new( + params: PublicKeyAggregatorParams, + state: Persistable, + ) -> Self { + PublicKeyAggregator { + fhe: params.fhe, + bus: params.bus, + e3_id: params.e3_id, + state, + params_preset: params.params_preset, + committee_size: params.committee_size, + early_dkg_proofs: Vec::new(), + } + } +} + +#[path = "effects/mod.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +#[cfg(test)] +#[path = "tests/mod.rs"] +mod tests; diff --git a/crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs b/crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs new file mode 100644 index 0000000000..7980ee5125 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/effects/aggregate_dkg_proofs.rs @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Dispatch the final DKG aggregation proof once all prerequisites exist. + +use super::super::*; + +impl PublicKeyAggregator { + /// Dispatch [`ZkRequest::DkgAggregation`] once C5, all honest NodeFold proofs, and the + /// streaming nodes_fold are all ready. + pub(in crate::actors::publickey_aggregator) fn try_dispatch_dkg_aggregation( + &mut self, + ec: &EventContext, + ) -> Result<()> { + let state = self.state.get(); + let Some(PublicKeyAggregatorState::GeneratingC5Proof { + party_nodes, + dkg_node_proofs, + honest_party_ids, + c5_proof_pending, + dkg_aggregation_correlation, + dkg_aggregated_proof, + circuit_committee_n, + circuit_committee_h, + nodes_fold_accumulator, + nodes_fold_completed_slots, + .. + }) = state.as_ref() + else { + return Ok(()); + }; + + let Some(c5_proof) = c5_proof_pending.as_ref() else { + return Ok(()); + }; + + if dkg_aggregation_correlation.is_some() || dkg_aggregated_proof.is_some() { + return Ok(()); + } + + let all_honest_proofs_present = honest_party_ids + .iter() + .all(|id| dkg_node_proofs.contains_key(id)); + if !all_honest_proofs_present { + return Ok(()); + } + + // `proof_aggregation_enabled` is an E3-level flag shared by all nodes, so honest-party + // proofs should be uniformly Some (aggregation on) or uniformly None (aggregation off). + // A mixed bag would silently truncate the dispatched request below; reject it explicitly. + let some_count = honest_party_ids + .iter() + .filter(|id| { + dkg_node_proofs + .get(id) + .map(Option::is_some) + .unwrap_or(false) + }) + .count(); + if some_count != 0 && some_count != honest_party_ids.len() { + error!( + "PublicKeyAggregator: mixed Some/None DKG node proofs across honest parties \ + ({some_count} of {} present); failing E3 {}", + honest_party_ids.len(), + self.e3_id + ); + self.bus.publish( + E3Failed { + e3_id: self.e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGInvalidShares, + }, + ec.clone(), + )?; + self.state.try_mutate(ec, |state| { + let PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation: _, + dkg_aggregated_proof, + c5_proof_pending: _, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + } = state + else { + return Ok(state); + }; + + Ok(PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation: None, + dkg_aggregated_proof, + c5_proof_pending: None, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + }) + })?; + return Ok(()); + } + + let mut pairs: Vec<_> = dkg_node_proofs + .iter() + .filter(|(pid, _)| honest_party_ids.contains(pid)) + .filter_map(|(pid, p)| p.as_ref().map(|proof| (*pid, proof.clone()))) + .collect(); + pairs.sort_by_key(|(pid, _)| *pid); + let party_ids: Vec = pairs.iter().map(|(pid, _)| *pid).collect(); + let node_fold_proofs: Vec = pairs.into_iter().map(|(_, p)| p).collect(); + info!( + "ORDER-DEBUG dispatch DkgAggregation: honest_party_ids(submission-idx)={:?} \ + dkg_node_proofs_keys(real party_id from DKGRecursiveAggregationComplete)={:?} \ + party_ids_passed_to_circuit={:?}", + honest_party_ids.iter().collect::>(), + { + let mut k: Vec = dkg_node_proofs.keys().copied().collect(); + k.sort(); + k + }, + party_ids + ); + + if node_fold_proofs.is_empty() { + // Proof aggregation disabled. Do NOT call `try_publish_complete` here — it + // is the most common entry into this method, so re-entering it would create + // unbounded mutual recursion (stack overflow in deployed nodes). + info!("PublicKeyAggregator: proof aggregation disabled — skipping DkgAggregation"); + return Ok(()); + } + + // Streaming fold must be complete before dispatching the final aggregation. + let fold_complete = *nodes_fold_completed_slots == honest_party_ids.len() as u32; + if !fold_complete { + return Ok(()); + } + let precomputed_fold = nodes_fold_accumulator.clone(); + + // Build the FULL committee address vector (length N) in ascending party_id order. + // The DKG aggregator circuit's `committee_members: [Field; N_PARTIES]` is the + // committee-hash preimage; passing only the H honest subset would silently + // hash a shorter array and diverge from on-chain `keccak(topNodes)`. + let mut full_committee_party_ids: Vec = party_nodes.keys().copied().collect(); + full_committee_party_ids.sort(); + let committee_addresses = + committee_addresses_in_party_order(&full_committee_party_ids, party_nodes)?; + #[cfg(debug_assertions)] + { + debug_assert_eq!( + committee_addresses.len(), + *circuit_committee_n, + "DkgAggregator committee_addresses must have N entries (full topNodes)" + ); + debug_assert_eq!( + party_ids.len(), + *circuit_committee_h, + "DkgAggregator party_ids must have H entries (honest set)" + ); + } + + let corr = CorrelationId::new(); + self.bus.publish( + ComputeRequest::zk( + ZkRequest::DkgAggregation(DkgAggregationRequest { + node_fold_proofs, + nodes_fold_proof: precomputed_fold, + c5_proof: c5_proof.clone(), + party_ids, + committee_addresses, + params_preset: self.params_preset, + committee_size: self.committee_size, + }), + corr, + self.e3_id.clone(), + ), + ec.clone(), + )?; + + self.state.try_mutate(ec, |state| { + let PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation: _, + dkg_aggregated_proof, + c5_proof_pending, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + } = state + else { + return Ok(state); + }; + Ok(PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation: Some(corr), + dkg_aggregated_proof, + c5_proof_pending, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + }) + })?; + Ok(()) + } +} diff --git a/crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs b/crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs new file mode 100644 index 0000000000..aaf23321fb --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/effects/aggregate_public_key.rs @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Correlate C5 and per-node DKG fold artifacts. + +use super::super::*; + +impl PublicKeyAggregator { + pub(in crate::actors::publickey_aggregator) fn handle_pk_aggregation_proof_signed( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + + if msg.e3_id != self.e3_id { + return Ok(()); + } + + if !matches!( + self.state.get(), + Some(PublicKeyAggregatorState::GeneratingC5Proof { .. }) + ) { + return Err(anyhow::anyhow!( + "handle_pk_aggregation_proof_signed called outside GeneratingC5Proof state" + )); + } + + info!("C5 proof signed — waiting for cross-node DKG fold to complete..."); + + let c5_proof = msg.signed_proof.payload.proof.clone(); + self.state.try_mutate(&ec, |state| { + let PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + .. + } = state + else { + return Ok(state); + }; + Ok(PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + c5_proof_pending: Some(c5_proof), + last_ec: Some(ec.clone()), + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + }) + })?; + self.try_publish_complete() + } + + // -- Cross-node DKG proof aggregation -------------------------------------------------- + + pub(in crate::actors::publickey_aggregator) fn handle_dkg_recursive_aggregation_complete( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + + if msg.e3_id != self.e3_id { + return Ok(()); + } + + let state = self.state.get(); + let Some(PublicKeyAggregatorState::GeneratingC5Proof { + party_nodes, + dkg_node_proofs, + honest_party_ids, + circuit_committee_n, + circuit_committee_h, + .. + }) = state.as_ref() + else { + info!( + "PublicKeyAggregator: early DKG proof from party {} — buffering until GeneratingC5Proof", + msg.party_id + ); + self.early_dkg_proofs.push(TypedEvent::new(msg, ec)); + return Ok(()); + }; + if dkg_node_proofs.contains_key(&msg.party_id) { + warn!( + "Duplicate DKGRecursiveAggregationComplete for party {} — ignoring", + msg.party_id + ); + return Ok(()); + } + + if honest_party_ids.contains(&msg.party_id) { + let Some(expected_node) = party_nodes.get(&msg.party_id) else { + warn!( + party_id = msg.party_id, + "DKG fold from party without registered node address — rejecting" + ); + return Ok(()); + }; + // Proof aggregation OFF: nodes emit `DKGRecursiveAggregationComplete` + // with `proof=None` and `attestation=None`. Accept it so + // `try_publish_complete` can detect `all_proofs_are_none` and publish. + // Proof aggregation ON: both must be present and verified together. + match (&msg.aggregated_proof, &msg.fold_attestation) { + (None, None) => { + // no-aggregation mode — skip attestation verification + } + (Some(proof), Some(attestation)) => { + let meta = self.params_preset.metadata(); + let committee_n = *circuit_committee_n; + let committee_h = *circuit_committee_h; + let n_moduli = meta.num_moduli; + if committee_n == 0 || committee_h == 0 { + warn!( + party_id = msg.party_id, + "DKG fold attestation verify skipped — circuit committee dims unset" + ); + return Ok(()); + } + if let Err(e) = verify_dkg_fold_attestation( + &self.e3_id, + msg.party_id, + proof, + attestation, + expected_node, + committee_n, + committee_h, + n_moduli, + ) { + warn!( + party_id = msg.party_id, + error = %e, + "DKG fold attestation verification failed — rejecting" + ); + return Ok(()); + } + } + (Some(_), None) => { + warn!( + party_id = msg.party_id, + "DKG fold has proof but missing attestation — rejecting (attribution)" + ); + return Ok(()); + } + (None, Some(_)) => { + warn!( + party_id = msg.party_id, + "DKG fold has attestation but missing proof — rejecting" + ); + return Ok(()); + } + } + } + + info!( + "PublicKeyAggregator: buffered DKG proof from party {} (buffered={})", + msg.party_id, + dkg_node_proofs.len() + 1 + ); + + self.state.try_mutate(&ec, |state| { + let PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + mut dkg_node_proofs, + mut dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + c5_proof_pending, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + last_ec: _, + } = state + else { + return Ok(state); + }; + dkg_node_proofs.insert(msg.party_id, msg.aggregated_proof); + if let Some(attestation) = msg.fold_attestation.clone() { + dkg_fold_attestations.insert(msg.party_id, attestation); + } + Ok(PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + c5_proof_pending, + last_ec: Some(ec.clone()), + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + }) + })?; + + self.try_dispatch_nodes_fold_step(&ec) + } +} diff --git a/crates/aggregator/src/public_key_aggregation/effects/fold_node_proofs.rs b/crates/aggregator/src/public_key_aggregation/effects/fold_node_proofs.rs new file mode 100644 index 0000000000..0dbb6beba4 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/effects/fold_node_proofs.rs @@ -0,0 +1,204 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Stream the canonical honest-party NodeFold accumulator. + +use super::super::*; + +impl PublicKeyAggregator { + /// Dispatch the next [`ZkRequest::NodesFoldStep`] if the next slot's proof is buffered + /// and no step is currently in flight. When all H slots are done, calls + /// [`try_dispatch_dkg_aggregation`]. + pub(in crate::actors::publickey_aggregator) fn try_dispatch_nodes_fold_step( + &mut self, + ec: &EventContext, + ) -> Result<()> { + let state = self.state.get(); + let Some(PublicKeyAggregatorState::GeneratingC5Proof { + dkg_node_proofs, + honest_party_ids, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + dkg_aggregation_correlation, + dkg_aggregated_proof, + .. + }) = state.as_ref() + else { + return Ok(()); + }; + + if nodes_fold_step_correlation.is_some() + || dkg_aggregation_correlation.is_some() + || dkg_aggregated_proof.is_some() + { + return Ok(()); + } + + let next_slot = *nodes_fold_completed_slots; + let total_slots = honest_party_ids.len(); + + if next_slot as usize >= total_slots { + return self.try_dispatch_dkg_aggregation(ec); + } + + let Some(&party_id) = honest_party_ids.iter().nth(next_slot as usize) else { + return Ok(()); + }; + + let Some(Some(inner_proof)) = dkg_node_proofs.get(&party_id) else { + return Ok(()); + }; + + let inner_proof = inner_proof.clone(); + let prior_accumulator = nodes_fold_accumulator.clone(); + + let corr = CorrelationId::new(); + self.bus.publish( + ComputeRequest::zk( + ZkRequest::NodesFoldStep(NodesFoldStepRequest { + inner_proof, + prior_accumulator, + slot_index: next_slot, + total_slots, + e3_id: self.e3_id.to_string(), + params_preset: self.params_preset, + committee_size: self.committee_size, + }), + corr, + self.e3_id.clone(), + ), + ec.clone(), + )?; + + info!( + "PublicKeyAggregator: dispatched NodesFoldStep slot={}/{} for E3 {}", + next_slot, total_slots, self.e3_id + ); + + self.state.try_mutate(ec, |state| { + let PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + c5_proof_pending, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation: _, + } = state + else { + return Ok(state); + }; + Ok(PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + c5_proof_pending, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation: Some(corr), + }) + })?; + Ok(()) + } + + /// Handle a completed [`ZkResponse::NodesFoldStep`]: advance the accumulator and dispatch + /// the next fold step (or the final DkgAggregation when all H slots are done). + pub(in crate::actors::publickey_aggregator) fn handle_nodes_fold_step_response( + &mut self, + correlation_id: CorrelationId, + accumulator_proof: Proof, + ) -> Result<()> { + let state = self.state.get(); + let Some(PublicKeyAggregatorState::GeneratingC5Proof { + nodes_fold_step_correlation, + nodes_fold_completed_slots, + last_ec, + .. + }) = state.as_ref() + else { + return Ok(()); + }; + + if nodes_fold_step_correlation.as_ref() != Some(&correlation_id) { + return Ok(()); + } + + let completed = nodes_fold_completed_slots + 1; + let Some(ec) = last_ec.clone() else { + return Err(anyhow::anyhow!( + "No EventContext for NodesFoldStep response" + )); + }; + + info!( + "PublicKeyAggregator: NodesFoldStep complete (slot {} done) for E3 {}", + completed - 1, + self.e3_id + ); + + self.state.try_mutate_without_context(|state| { + let PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + c5_proof_pending, + last_ec, + nodes_fold_step_correlation: _, + .. + } = state + else { + return Ok(state); + }; + Ok(PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + c5_proof_pending, + last_ec, + nodes_fold_accumulator: Some(accumulator_proof), + nodes_fold_completed_slots: completed, + nodes_fold_step_correlation: None, + }) + })?; + + self.try_dispatch_nodes_fold_step(&ec) + } +} diff --git a/crates/aggregator/src/public_key_aggregation/effects/handle_compute_results.rs b/crates/aggregator/src/public_key_aggregation/effects/handle_compute_results.rs new file mode 100644 index 0000000000..a2b06d8d8b --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/effects/handle_compute_results.rs @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Correlate compute results and map terminal worker failures. + +use super::super::*; + +impl PublicKeyAggregator { + pub(in crate::actors::publickey_aggregator) fn handle_compute_response( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, _ec) = msg.into_components(); + if msg.e3_id != self.e3_id { + return Ok(()); + } + match msg.response { + ComputeResponseKind::Zk(ZkResponse::NodesFoldStep(resp)) => { + self.handle_nodes_fold_step_response(msg.correlation_id, resp.accumulator_proof)?; + } + ComputeResponseKind::Zk(ZkResponse::DkgAggregation(resp)) => { + let state = self.state.get(); + let Some(PublicKeyAggregatorState::GeneratingC5Proof { last_ec, .. }) = + state.as_ref() + else { + return Ok(()); + }; + let Some(_ec) = last_ec.clone() else { + return Err(anyhow::anyhow!( + "No EventContext for DkgAggregation response" + )); + }; + self.state.try_mutate_without_context(|state| { + let PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + c5_proof_pending, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + } = state + else { + return Ok(state); + }; + if dkg_aggregation_correlation.as_ref() != Some(&msg.correlation_id) { + return Ok(PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + c5_proof_pending, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + }); + } + Ok(PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation: None, + dkg_aggregated_proof: Some(resp.proof.clone()), + c5_proof_pending, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + }) + })?; + self.try_publish_complete()?; + } + _ => {} + } + Ok(()) + } + + pub(in crate::actors::publickey_aggregator) fn handle_compute_request_error( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + if msg.request().e3_id != self.e3_id { + return Ok(()); + } + + let matched_nodes_fold_step = matches!( + self.state.get(), + Some(PublicKeyAggregatorState::GeneratingC5Proof { + nodes_fold_step_correlation, + .. + }) if nodes_fold_step_correlation.as_ref() == Some(msg.correlation_id()) + ); + + if matched_nodes_fold_step { + error!( + "PublicKeyAggregator: NodesFoldStep failed for E3 {}: {:?}", + self.e3_id, + msg.get_err() + ); + self.bus.publish( + E3Failed { + e3_id: self.e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGInvalidShares, + }, + ec.clone(), + )?; + self.state.try_mutate(&ec, |state| { + let PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + c5_proof_pending: _, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation: _, + } = state + else { + return Ok(state); + }; + Ok(PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation, + dkg_aggregated_proof, + c5_proof_pending: None, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation: None, + }) + })?; + return Ok(()); + } + + let matched_dkg_aggregation = matches!( + self.state.get(), + Some(PublicKeyAggregatorState::GeneratingC5Proof { + dkg_aggregation_correlation, + .. + }) if dkg_aggregation_correlation.as_ref() == Some(msg.correlation_id()) + ); + + if !matched_dkg_aggregation { + return Ok(()); + } + + error!( + "PublicKeyAggregator: DkgAggregation failed for E3 {}: {:?}", + self.e3_id, + msg.get_err() + ); + + self.bus.publish( + E3Failed { + e3_id: self.e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGInvalidShares, + }, + ec.clone(), + )?; + + self.state.try_mutate(&ec, |state| { + let PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation: _, + dkg_aggregated_proof, + c5_proof_pending: _, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + } = state + else { + return Ok(state); + }; + + Ok(PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + keyshare_bytes, + nodes, + party_nodes, + dkg_node_proofs, + dkg_fold_attestations, + honest_party_ids, + dishonest_parties, + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation: None, + dkg_aggregated_proof, + c5_proof_pending: None, + last_ec, + nodes_fold_accumulator, + nodes_fold_completed_slots, + nodes_fold_step_correlation, + }) + })?; + + Ok(()) + } +} diff --git a/crates/aggregator/src/public_key_aggregation/effects/mod.rs b/crates/aggregator/src/public_key_aggregation/effects/mod.rs new file mode 100644 index 0000000000..23b01b30d7 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/effects/mod.rs @@ -0,0 +1,28 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Effect execution for public-key aggregation. + +use super::*; + +mod aggregate_dkg_proofs; +mod aggregate_public_key; +mod fold_node_proofs; +mod handle_compute_results; +mod publish_result; +mod verify_key_proofs; + +impl PublicKeyAggregator { + pub fn handle_member_expelled( + &mut self, + node: &str, + ec: &EventContext, + ) -> Result<()> { + self.state.try_mutate(ec, |state| { + PublicKeyAggregation::handle_member_expelled(state, node) + }) + } +} diff --git a/crates/aggregator/src/public_key_aggregation/effects/publish_result.rs b/crates/aggregator/src/public_key_aggregation/effects/publish_result.rs new file mode 100644 index 0000000000..c4f2ac5894 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/effects/publish_result.rs @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Publish the completed public-key artifact and persist terminal state. + +use super::super::*; + +impl PublicKeyAggregator { + /// Publish `PublicKeyAggregated` when C5 (non-ZK recursive) and, if applicable, the EVM DkgAggregator proof are ready (or aggregation skipped). + pub(in crate::actors::publickey_aggregator) fn try_publish_complete(&mut self) -> Result<()> { + if let Some(ec) = self.state.get().and_then(|s| { + if let PublicKeyAggregatorState::GeneratingC5Proof { last_ec, .. } = &s { + last_ec.clone() + } else { + None + } + }) { + self.try_dispatch_dkg_aggregation(&ec)?; + } + + let PublicKeyAggregatorState::GeneratingC5Proof { + public_key, + nodes, + party_nodes, + dkg_fold_attestations, + honest_party_ids, + c5_proof_pending, + dkg_aggregated_proof, + dkg_aggregation_correlation: _, + last_ec, + .. + } = self + .state + .get() + .ok_or_else(|| anyhow::anyhow!("Expected GeneratingC5Proof state"))? + else { + return Ok(()); + }; + + let Some(c5_proof) = c5_proof_pending.as_ref() else { + return Ok(()); + }; + + let all_proofs_are_none = self + .state + .get() + .and_then(|s| { + if let PublicKeyAggregatorState::GeneratingC5Proof { + dkg_node_proofs, + honest_party_ids, + .. + } = &s + { + let all_present = honest_party_ids + .iter() + .all(|id| dkg_node_proofs.contains_key(id)); + Some(all_present && dkg_node_proofs.values().all(|p| p.is_none())) + } else { + None + } + }) + .unwrap_or(false); + + if !all_proofs_are_none && dkg_aggregated_proof.is_none() { + return Ok(()); + } + + let ec = last_ec + .clone() + .ok_or_else(|| anyhow::anyhow!("No EventContext for publish"))?; + + info!( + "Publishing PublicKeyAggregated (dkg_evm_proof={})", + if dkg_aggregated_proof.is_some() { + "present" + } else { + "skipped" + } + ); + + let pk_commitment = extract_pk_commitment(c5_proof)?; + + // Full committee (N entries) — used by on-chain `committee_hash` binding. + let mut full_committee_party_ids: Vec = party_nodes.keys().copied().collect(); + full_committee_party_ids.sort(); + let committee_addresses = + committee_addresses_in_party_order(&full_committee_party_ids, &party_nodes)?; + + // Honest subset (H entries) — used by downstream actors for share-collection gating. + let honest_party_ids_vec: Vec = honest_party_ids.iter().copied().collect(); + let honest_committee_addresses = + committee_addresses_in_party_order(&honest_party_ids_vec, &party_nodes)?; + + let dkg_attestation_bundle = match dkg_aggregated_proof.as_ref() { + Some(_) => { + let bundle = e3_zk_prover::encode_dkg_attestation_bundle( + &honest_party_ids, + &party_nodes, + &dkg_fold_attestations, + )?; + Some(ArcBytes::from_bytes(&bundle)) + } + None => None, + }; + + let event = PublicKeyAggregated { + pubkey: public_key.clone(), + e3_id: self.e3_id.clone(), + nodes: nodes.clone(), + committee_addresses: committee_addresses.clone(), + honest_committee_addresses: honest_committee_addresses.clone(), + pk_commitment, + dkg_aggregator_proof: dkg_aggregated_proof.clone(), + dkg_attestation_bundle, + }; + self.bus.publish(event, ec.clone())?; + + self.state.try_mutate(&ec, |_| { + Ok(PublicKeyAggregatorState::Complete { + public_key, + keyshares: OrderedSet::new(), + nodes, + committee_addresses, + honest_committee_addresses, + }) + })?; + + Ok(()) + } +} diff --git a/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs b/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs new file mode 100644 index 0000000000..e6a6e47537 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/effects/verify_key_proofs.rs @@ -0,0 +1,290 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! C1 verification and honest-keyshare selection. + +use super::*; + +impl PublicKeyAggregator { + pub fn add_keyshare( + &mut self, + keyshare: ArcBytes, + node: String, + party_id: u64, + c1_proof: Option, + ec: &EventContext, + ) -> Result<()> { + self.state.try_mutate(ec, |state| { + PublicKeyAggregation::add_keyshare( + state, + keyshare.clone(), + node.clone(), + party_id, + c1_proof.clone(), + ) + }) + } + + pub(in crate::actors::publickey_aggregator) fn dispatch_c1_verification( + &mut self, + submission_order: &[(u64, String, ArcBytes)], + c1_proofs: &[Option], + ec: EventContext, + ) -> Result<()> { + let C1Dispatch { + party_proofs, + no_proof_parties, + } = PublicKeyAggregation::plan_c1_dispatch(submission_order, c1_proofs); + + // Store no-proof parties in state for the response handler + if !no_proof_parties.is_empty() { + self.state.try_mutate(&ec, |mut state| { + if let PublicKeyAggregatorState::VerifyingC1 { + no_proof_parties: ref mut stored, + .. + } = state + { + *stored = no_proof_parties.clone(); + } + Ok(state) + })?; + } + + if party_proofs.is_empty() { + return Err(anyhow::anyhow!( + "No C1 proofs to verify — all keyshares must include a signed C1 proof" + )); + } + + info!( + "Dispatching C1 proof verification for {} parties ({} missing proofs)", + party_proofs.len(), + no_proof_parties.len() + ); + + self.bus.publish( + ShareVerificationDispatched { + e3_id: self.e3_id.clone(), + kind: VerificationKind::PkGenerationProofs, + share_proofs: party_proofs, + decryption_proofs: vec![], + pre_dishonest: no_proof_parties.into_iter().collect(), + params_preset: self.params_preset, + committee_size: self.committee_size, + }, + ec, + )?; + Ok(()) + } + + pub(in crate::actors::publickey_aggregator) fn handle_c1_verification_complete( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + + if msg.kind != VerificationKind::PkGenerationProofs { + return Ok(()); + } + + if msg.e3_id != self.e3_id { + return Ok(()); + } + + let PublicKeyAggregatorState::VerifyingC1 { + submission_order, + threshold_m, + circuit_committee_n, + circuit_committee_h, + c1_proofs, + .. + } = self + .state + .get() + .ok_or_else(|| anyhow::anyhow!("Expected VerifyingC1 state"))? + else { + return Err(anyhow::anyhow!( + "handle_c1_verification_complete called outside VerifyingC1 state" + )); + }; + + let mut dishonest_parties = msg.dishonest_parties.clone(); + let collected = submission_order.len(); + let circuit_h = circuit_committee_h; + + // Retain full N committee roster (party_id → node address) for the DKG aggregator + // `committee_members` input, which must cover all `topNodes` regardless of honesty. + let full_submission_order: Vec<(u64, String, ArcBytes)> = submission_order.clone(); + + // Filter out parties that failed C1 ZK verification. Keyed by the real + // sortition party_id carried in `submission_order`, not arrival index. + let mut honest_entries: Vec<(u64, String, ArcBytes, Option)> = + submission_order + .into_iter() + .zip(c1_proofs) + .filter(|((pid, _, _), _)| !dishonest_parties.contains(pid)) + .map(|((pid, node, ks), c1)| (pid, node, ks, c1)) + .collect(); + + // Cross-check: verify each party's keyshare matches their C1 pk_commitment. + // Parties that fail are marked dishonest and reported via SignedProofFailed. + let audit = check_c1_keyshare_commitments(&honest_entries, &self.fhe); + for party_id in &audit.missing_proof { + dishonest_parties.insert(*party_id); + } + + // Emit SignedProofFailed for each commitment-mismatched party + for (party_id, signed_proof) in &audit.mismatched { + dishonest_parties.insert(*party_id); + match signed_proof.recover_address() { + Ok(faulting_node) => { + if let Err(e) = self.bus.publish( + SignedProofFailed { + e3_id: self.e3_id.clone(), + faulting_node, + proof_type: ProofType::C1PkGeneration, + signed_payload: signed_proof.clone(), + }, + ec.clone(), + ) { + error!("Failed to publish SignedProofFailed: {e}"); + } + } + Err(e) => warn!( + "Could not recover address from C1 proof for party {}: {e}", + party_id + ), + } + } + + if !audit.mismatched.is_empty() { + warn!( + "C1 commitment mismatch for {} parties — filtering before aggregation", + audit.mismatched.len() + ); + // Re-filter honest_entries after commitment check + honest_entries.retain(|(pid, _, _, _)| !dishonest_parties.contains(pid)); + } + + // Sort, fail-closed below H, cap to the H lowest party_ids, and fail when + // <= threshold_m remain. All pure decision logic lives in the service; the + // actor only publishes E3Failed on the Fail outcome. + let (honest_entries, honest_party_ids) = match PublicKeyAggregation::select_honest_set( + &self.e3_id, + honest_entries, + &dishonest_parties, + circuit_h, + threshold_m, + collected, + ) { + HonestSelection::Fail => { + self.bus.publish( + E3Failed { + e3_id: self.e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGInvalidShares, + }, + ec, + )?; + return Ok(()); + } + HonestSelection::Proceed { + honest_entries, + honest_party_ids, + } => (honest_entries, honest_party_ids), + }; + + let (honest_keyshares, honest_nodes): (Vec, Vec) = honest_entries + .iter() + .map(|(_, node, ks, _)| (ks.clone(), node.clone())) + .unzip(); + + debug_assert_eq!( + honest_party_ids.len(), + honest_keyshares.len(), + "honest roster and keyshare payload lengths must match" + ); + + // Synchronous aggregation + info!( + "Aggregating public key from {} honest shares...", + honest_keyshares.len() + ); + let honest_keyshares_set = OrderedSet::from(honest_keyshares.clone()); + let pubkey = self.fhe.get_aggregate_public_key(GetAggregatePublicKey { + keyshares: honest_keyshares_set.clone(), + })?; + + let committee_h = honest_keyshares.len(); + let honest_nodes_set = OrderedSet::from(honest_nodes.clone()); + // Feed keyshares to C5 in ascending party_id order so that + // `c5_public[i]` (pk_commitment of the i-th input keyshare) matches + // party_ids[i] and the row-i node_fold pk bound by dkg_aggregator.nr. + // `honest_keyshares` preserves the submission-index (== party_id) order + // from `honest_entries`; do NOT sort by byte content. + let keyshare_bytes: Vec = honest_keyshares.clone(); + + let pubkey = ArcBytes::from_bytes(&pubkey); + info!("Publishing PkAggregationProofPending for C5 proof generation..."); + self.bus.publish( + PkAggregationProofPending { + e3_id: self.e3_id.clone(), + proof_request: PkAggregationProofRequest { + keyshare_bytes: keyshare_bytes.clone(), + aggregated_pk_bytes: pubkey.clone(), + params_preset: self.params_preset, + // C5 witness uses `committee_h` keyshares; artifact lookup needs canonical (N, H, T). + committee_n: circuit_committee_n, + committee_h, + committee_threshold: threshold_m, + }, + public_key: pubkey.clone(), + nodes: honest_nodes_set.clone(), + }, + ec.clone(), + )?; + + // `party_nodes` covers the FULL registered committee (all N keyshare submitters), + // not just the H honest set. The DKG aggregator circuit binds `committee_members` + // to on-chain `topNodes` which always carries the full committee — so we must keep + // the dishonest addresses available here to build the N-sized address vector. + // `submission_order` here is the unfiltered list captured pre–C1 verification + // (the original `VerifyingC1.submission_order`); `honest_entries` is the H subset. + let party_nodes: HashMap = full_submission_order + .iter() + .map(|(pid, node, _)| (*pid, node.clone())) + .collect(); + + let circuit_committee_h = circuit_h; + self.state.try_mutate(&ec, |_| { + Ok(PublicKeyAggregatorState::GeneratingC5Proof { + public_key: pubkey.clone(), + keyshare_bytes, + nodes: honest_nodes_set, + party_nodes, + dkg_node_proofs: HashMap::new(), + dkg_fold_attestations: HashMap::new(), + honest_party_ids: honest_party_ids.clone(), + dishonest_parties: dishonest_parties.clone(), + circuit_committee_n, + circuit_committee_h, + dkg_aggregation_correlation: None, + dkg_aggregated_proof: None, + c5_proof_pending: None, + last_ec: Some(ec.clone()), + nodes_fold_accumulator: None, + nodes_fold_completed_slots: 0, + nodes_fold_step_correlation: None, + }) + })?; + + // Replay any DKG proofs that arrived before we entered GeneratingC5Proof. + let early = std::mem::take(&mut self.early_dkg_proofs); + for event in early { + self.handle_dkg_recursive_aggregation_complete(event)?; + } + + self.try_dispatch_dkg_aggregation(&ec)?; + + Ok(()) + } +} diff --git a/crates/aggregator/src/public_key_aggregation/handlers.rs b/crates/aggregator/src/public_key_aggregation/handlers.rs new file mode 100644 index 0000000000..d59c2db9d7 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/handlers.rs @@ -0,0 +1,212 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Actix lifecycle and message routing for public-key aggregation. + +use super::*; + +impl Actor for PublicKeyAggregator { + type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } +} + +impl Handler for PublicKeyAggregator { + type Result = (); + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + match msg { + InterfoldEventData::KeyshareCreated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ShareVerificationComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::PkAggregationProofSigned(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::DKGRecursiveAggregationComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ComputeResponse(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ComputeRequestError(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::E3RequestComplete(_) => self.notify_sync(ctx, Die), + InterfoldEventData::CommitteeMemberExpelled(data) => { + // Only process raw events from chain (party_id not yet resolved). + if data.party_id.is_some() { + return; + } + + let node_addr = data.node.to_string(); + + if data.e3_id != self.e3_id { + error!("Wrong e3_id sent to PublicKeyAggregator for expulsion. This should not happen."); + return; + } + + info!( + "PublicKeyAggregator: committee member expelled: {} for e3_id={}", + node_addr, data.e3_id + ); + trap(EType::PublickeyAggregation, &self.bus.with_ec(&ec), || { + let was_collecting = matches!( + self.state.get(), + Some(PublicKeyAggregatorState::Collecting { .. }) + ); + + self.handle_member_expelled(&node_addr, &ec)?; + + // If we just transitioned to VerifyingC1, dispatch C1 verification + // using the c1_proofs now stored in the VerifyingC1 state (already + // cleaned of the expelled node's entry). + if was_collecting { + if let Some(PublicKeyAggregatorState::VerifyingC1 { + submission_order, + c1_proofs, + .. + }) = self.state.get() + { + self.dispatch_c1_verification( + &submission_order, + &c1_proofs, + ec.clone(), + )?; + } + } + Ok(()) + }); + } + _ => (), + }; + } +} + +impl Handler> for PublicKeyAggregator { + type Result = (); + + fn handle( + &mut self, + event: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (event, ec) = event.into_components(); + trap(EType::PublickeyAggregation, &self.bus.with_ec(&ec), || { + let e3_id = event.e3_id.clone(); + let pubkey = event.pubkey.clone(); + let node = event.node.clone(); + let party_id = event.party_id; + let c1_proof = event.signed_pk_generation_proof.clone(); + + if e3_id != self.e3_id { + error!("Wrong e3_id sent to aggregator. This should not happen."); + return Ok(()); + } + + self.add_keyshare(pubkey, node, party_id, c1_proof, &ec)?; + + // If we just transitioned to VerifyingC1, dispatch verification + // using c1_proofs stored in the new state. + if let Some(PublicKeyAggregatorState::VerifyingC1 { + submission_order, + c1_proofs, + .. + }) = self.state.get() + { + self.dispatch_c1_verification(&submission_order, &c1_proofs, ec)?; + } + + Ok(()) + }) + } +} + +impl Handler> for PublicKeyAggregator { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PublickeyAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_c1_verification_complete(msg), + ) + } +} + +impl Handler> for PublicKeyAggregator { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PublickeyAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_pk_aggregation_proof_signed(msg), + ) + } +} + +impl Handler> for PublicKeyAggregator { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PublickeyAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_dkg_recursive_aggregation_complete(msg), + ) + } +} + +impl Handler> for PublicKeyAggregator { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PublickeyAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_compute_response(msg), + ) + } +} + +impl Handler> for PublicKeyAggregator { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::PublickeyAggregation, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_compute_request_error(msg), + ) + } +} + +impl Handler for PublicKeyAggregator { + type Result = (); + fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { + ctx.stop(); + } +} diff --git a/crates/aggregator/src/actors/keyshare_created_filter_buffer.rs b/crates/aggregator/src/public_key_aggregation/keyshare_buffer.rs similarity index 100% rename from crates/aggregator/src/actors/keyshare_created_filter_buffer.rs rename to crates/aggregator/src/public_key_aggregation/keyshare_buffer.rs diff --git a/crates/aggregator/src/public_key_aggregation/state.rs b/crates/aggregator/src/public_key_aggregation/state.rs new file mode 100644 index 0000000000..92692969f3 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/state.rs @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Persisted public-key aggregation state schema. + +use super::*; + +#[allow(clippy::large_enum_variant)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub enum PublicKeyAggregatorState { + Collecting { + /// Live committee size after expulsions; used only to decide when enough keyshares arrived. + threshold_n: usize, + threshold_m: usize, + /// Canonical on-chain / circuit committee size N (unchanged after expulsion). + circuit_committee_n: usize, + /// Canonical honest-party count H for circuit public IO. + circuit_committee_h: usize, + keyshares: OrderedSet, + /// C1 proofs collected from KeyshareCreated events, indexed by insertion order + /// (matches `submission_order`). + c1_proofs: Vec>, + seed: Seed, + nodes: OrderedSet, + /// Insertion-ordered (real sortition `party_id`, node, keyshare) triples. + /// Index matches `c1_proofs`. The real `party_id` comes from `KeyshareCreated` + /// and must be used for all downstream circuit slot indexing — arrival order + /// is non-deterministic and does not match sortition's committee position. + submission_order: Vec<(u64, String, ArcBytes)>, + }, + VerifyingC1 { + /// Insertion-ordered (party_id, node, keyshare) triples from Collecting. + submission_order: Vec<(u64, String, ArcBytes)>, + threshold_m: usize, + /// Canonical on-chain / circuit committee size N (for `committee_h` lookup). + circuit_committee_n: usize, + /// Canonical honest-party count H for circuit public IO. + circuit_committee_h: usize, + /// C1 proofs in the same insertion order as `submission_order`. + c1_proofs: Vec>, + /// Real party_ids that submitted no C1 proof — treated as dishonest. + no_proof_parties: Vec, + }, + GeneratingC5Proof { + public_key: ArcBytes, + keyshare_bytes: Vec, + nodes: OrderedSet, + /// Registered node address per sortition `party_id` for the **full** committee + /// (all `N` parties that submitted a keyshare, honest or not). Honest-only lookups + /// must intersect with `honest_party_ids`. + party_nodes: HashMap, + /// DKG recursive proofs per party (restart-critical). + dkg_node_proofs: HashMap>, + /// Per-party fold attestations collected with honest DKG folds. + dkg_fold_attestations: HashMap, + honest_party_ids: BTreeSet, + dishonest_parties: BTreeSet, + /// Circuit committee size N (NodeFold / DKG public IO layout). + circuit_committee_n: usize, + /// Circuit honest-party count H (NodeFold / DKG public IO layout). + circuit_committee_h: usize, + /// In-flight [`ZkRequest::DkgAggregation`], if any. + dkg_aggregation_correlation: Option, + /// Result from [`ZkResponse::DkgAggregation`] (replaces pairwise `FoldProofs`). + dkg_aggregated_proof: Option, + c5_proof_pending: Option, + last_ec: Option>, + /// Accumulated nodes_fold proof after `nodes_fold_completed_slots` streaming steps. + nodes_fold_accumulator: Option, + /// Number of slots folded so far; equals the next slot index to dispatch. + nodes_fold_completed_slots: u32, + /// Correlation ID of the in-flight [`ZkRequest::NodesFoldStep`], if any. + nodes_fold_step_correlation: Option, + }, + Complete { + public_key: ArcBytes, + keyshares: OrderedSet, + nodes: OrderedSet, + /// Ascending `party_id` order (matches on-chain `topNodes` after finalize sort). + committee_addresses: Vec
, + /// Honest subset (H entries) for decryption-share gating after restart. + honest_committee_addresses: Vec
, + }, +} + +impl PublicKeyAggregatorState { + /// Ordered `topNodes` when the committee set is known (post–committee formation). + pub fn committee_nodes(&self) -> Option<&OrderedSet> { + match self { + PublicKeyAggregatorState::Collecting { nodes, .. } if !nodes.is_empty() => Some(nodes), + PublicKeyAggregatorState::GeneratingC5Proof { nodes, .. } => Some(nodes), + PublicKeyAggregatorState::Complete { nodes, .. } => Some(nodes), + _ => None, + } + } + + pub fn committee_addresses(&self) -> Option<&[Address]> { + match self { + PublicKeyAggregatorState::Complete { + committee_addresses, + .. + } if !committee_addresses.is_empty() => Some(committee_addresses.as_slice()), + _ => None, + } + } + + pub fn honest_committee_addresses(&self) -> Option<&[Address]> { + match self { + PublicKeyAggregatorState::Complete { + honest_committee_addresses, + .. + } if !honest_committee_addresses.is_empty() => { + Some(honest_committee_addresses.as_slice()) + } + _ => None, + } + } + + pub fn init(threshold_n: usize, threshold_m: usize, seed: Seed) -> Self { + let circuit_committee_h = committee_h_for(threshold_m, threshold_n) + .unwrap_or_else(|e| panic!("invalid committee at init: {e}")); + PublicKeyAggregatorState::Collecting { + threshold_n, + threshold_m, + circuit_committee_n: threshold_n, + circuit_committee_h, + keyshares: OrderedSet::new(), + c1_proofs: Vec::new(), + seed, + nodes: OrderedSet::new(), + submission_order: Vec::new(), + } + } +} diff --git a/crates/aggregator/src/public_key_aggregation/tests/attestations.rs b/crates/aggregator/src/public_key_aggregation/tests/attestations.rs new file mode 100644 index 0000000000..2e51bdf309 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/tests/attestations.rs @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn honest_dkg_fold_without_attestation_is_not_buffered() -> Result<()> { + let correlation_id = CorrelationId::new(); + let mut initial_state = generating_c5_state(correlation_id); + let PublicKeyAggregatorState::GeneratingC5Proof { + ref mut party_nodes, + ref mut honest_party_ids, + .. + } = initial_state + else { + unreachable!(); + }; + honest_party_ids.insert(2); + party_nodes.insert(2, "0x70997970C51812dc3A010C7d01b50e0d17dc79C8".to_string()); + + let (mut aggregator, _history, e3_id) = build_public_key_aggregator(initial_state).await?; + let ec = test_ctx(DKGRecursiveAggregationComplete { + e3_id: e3_id.clone(), + party_id: 2, + aggregated_proof: Some(dummy_proof(CircuitName::NodeFold)), + fold_attestation: None, + }); + + aggregator.handle_dkg_recursive_aggregation_complete(TypedEvent::new( + DKGRecursiveAggregationComplete { + e3_id: e3_id.clone(), + party_id: 2, + aggregated_proof: Some(dummy_proof(CircuitName::NodeFold)), + fold_attestation: None, + }, + ec, + ))?; + + let Some(PublicKeyAggregatorState::GeneratingC5Proof { + dkg_node_proofs, .. + }) = aggregator.state.get() + else { + panic!("expected GeneratingC5Proof state"); + }; + assert!(!dkg_node_proofs.contains_key(&2)); + + Ok(()) +} + +#[actix::test] +async fn pk_aggregation_proof_pending_carries_canonical_committee_dims() -> Result<()> { + let (bus, rng, _seed, params, crp, _errors, history) = + get_common_setup(Some(BfvPreset::InsecureThreshold512.into()))?; + let e3_id = E3id::new("42", 1); + let fhe = Arc::new(Fhe::new(params, crp, rng)); + let (initial_state, threshold_n, threshold_m, circuit_h) = + verifying_c1_non_square_state(&fhe, &e3_id)?; + + let mut aggregator = PublicKeyAggregator::new( + PublicKeyAggregatorParams { + fhe, + bus, + e3_id: e3_id.clone(), + params_preset: BfvPreset::InsecureThreshold512, + committee_size: CiphernodesCommitteeSize::Micro, + }, + test_state(initial_state), + ); + + let dishonest: BTreeSet = (circuit_h as u64..threshold_n as u64).collect(); + aggregator.handle_c1_verification_complete(TypedEvent::new( + ShareVerificationComplete { + e3_id: e3_id.clone(), + kind: VerificationKind::PkGenerationProofs, + dishonest_parties: dishonest, + }, + test_ctx(ShareVerificationComplete { + e3_id: e3_id.clone(), + kind: VerificationKind::PkGenerationProofs, + dishonest_parties: BTreeSet::new(), + }), + ))?; + + let event = next_event(&history).await?; + assert!(matches!( + event.into_data(), + InterfoldEventData::PkAggregationProofPending(data) + if data.e3_id == e3_id + && data.proof_request.committee_n == threshold_n + && data.proof_request.committee_h == circuit_h + && data.proof_request.committee_threshold == threshold_m + && data.proof_request.keyshare_bytes.len() == circuit_h + )); + + Ok(()) +} diff --git a/crates/aggregator/src/public_key_aggregation/tests/failures.rs b/crates/aggregator/src/public_key_aggregation/tests/failures.rs new file mode 100644 index 0000000000..b7a4a59ed1 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/tests/failures.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn dkg_aggregation_compute_error_emits_e3_failed() -> Result<()> { + let correlation_id = CorrelationId::new(); + let (mut aggregator, history, e3_id) = + build_public_key_aggregator(generating_c5_state(correlation_id)).await?; + + let request = ComputeRequest::zk( + ZkRequest::DkgAggregation(DkgAggregationRequest { + node_fold_proofs: vec![dummy_proof(CircuitName::PkAggregation)], + nodes_fold_proof: None, + c5_proof: dummy_proof(CircuitName::PkAggregation), + party_ids: vec![0], + committee_addresses: vec!["0x0000000000000000000000000000000000000001" + .parse() + .expect("test address")], + params_preset: BfvPreset::InsecureThreshold512, + committee_size: CiphernodesCommitteeSize::Minimum, + }), + correlation_id, + e3_id.clone(), + ); + + aggregator.handle_compute_request_error(TypedEvent::new( + ComputeRequestError::new( + ComputeRequestErrorKind::Zk(ZkError::ProofGenerationFailed("boom".to_string())), + request, + ), + test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::None, + reason: FailureReason::None, + }), + ))?; + + let event = next_event(&history).await?; + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == e3_id + && data.failed_at_stage == E3Stage::CommitteeFinalized + && data.reason == FailureReason::DKGInvalidShares + )); + + let Some(PublicKeyAggregatorState::GeneratingC5Proof { + dkg_aggregation_correlation, + c5_proof_pending, + .. + }) = aggregator.state.get() + else { + panic!("expected GeneratingC5Proof state"); + }; + assert!(dkg_aggregation_correlation.is_none()); + assert!(c5_proof_pending.is_none()); + + Ok(()) +} + +#[actix::test] +async fn mixed_dkg_proofs_emit_e3_failed() -> Result<()> { + let correlation_id = CorrelationId::new(); + let mut initial_state = generating_c5_state(correlation_id); + let PublicKeyAggregatorState::GeneratingC5Proof { + ref mut dkg_aggregation_correlation, + ref mut dkg_node_proofs, + ref mut honest_party_ids, + .. + } = initial_state + else { + unreachable!(); + }; + *dkg_aggregation_correlation = None; + honest_party_ids.extend([0, 1]); + dkg_node_proofs.insert(0, Some(dummy_proof(CircuitName::PkAggregation))); + dkg_node_proofs.insert(1, None); + + let (mut aggregator, history, e3_id) = build_public_key_aggregator(initial_state).await?; + let ec = test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::None, + reason: FailureReason::None, + }); + + aggregator.try_dispatch_dkg_aggregation(&ec)?; + + let event = next_event(&history).await?; + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == e3_id + && data.failed_at_stage == E3Stage::CommitteeFinalized + && data.reason == FailureReason::DKGInvalidShares + )); + + let Some(PublicKeyAggregatorState::GeneratingC5Proof { + dkg_aggregation_correlation, + c5_proof_pending, + .. + }) = aggregator.state.get() + else { + panic!("expected GeneratingC5Proof state"); + }; + assert!(dkg_aggregation_correlation.is_none()); + assert!(c5_proof_pending.is_none()); + + Ok(()) +} diff --git a/crates/aggregator/src/public_key_aggregation/tests/mod.rs b/crates/aggregator/src/public_key_aggregation/tests/mod.rs new file mode 100644 index 0000000000..8392f0899c --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/tests/mod.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use e3_data::{AutoPersist, DataStore, InMemStore, Repository}; +use e3_events::{ + CircuitName, ComputeRequestErrorKind, HistoryCollector, ProofPayload, ProofType, TakeEvents, + Unsequenced, ZkError, +}; +use e3_test_helpers::get_common_setup; +use std::collections::BTreeSet; + +fn test_ctx(data: impl Into) -> EventContext { + EventContext::::from(data.into()).sequence(0) +} + +fn test_state(initial_state: PublicKeyAggregatorState) -> Persistable { + let repo = Repository::::new(DataStore::from_in_mem( + &InMemStore::new(false).start(), + )); + repo.to_connector().send(Some(initial_state)) +} + +fn dummy_proof(circuit: CircuitName) -> Proof { + Proof::new( + circuit, + ArcBytes::from_bytes(&[1]), + ArcBytes::from_bytes(&[2]), + ) +} + +fn generating_c5_state(correlation_id: CorrelationId) -> PublicKeyAggregatorState { + PublicKeyAggregatorState::GeneratingC5Proof { + public_key: ArcBytes::from_bytes(&[1, 2, 3]), + keyshare_bytes: Vec::new(), + nodes: OrderedSet::new(), + party_nodes: HashMap::new(), + dkg_node_proofs: HashMap::new(), + dkg_fold_attestations: HashMap::new(), + honest_party_ids: BTreeSet::new(), + dishonest_parties: BTreeSet::new(), + circuit_committee_n: 3, + circuit_committee_h: 3, + dkg_aggregation_correlation: Some(correlation_id), + dkg_aggregated_proof: None, + c5_proof_pending: Some(dummy_proof(CircuitName::PkAggregation)), + last_ec: None, + nodes_fold_accumulator: None, + nodes_fold_completed_slots: 0, + nodes_fold_step_correlation: None, + } +} + +async fn build_public_key_aggregator( + initial_state: PublicKeyAggregatorState, +) -> Result<( + PublicKeyAggregator, + Addr>, + E3id, +)> { + build_public_key_aggregator_with_committee(initial_state, CiphernodesCommitteeSize::Minimum) + .await +} + +async fn build_public_key_aggregator_with_committee( + initial_state: PublicKeyAggregatorState, + committee_size: CiphernodesCommitteeSize, +) -> Result<( + PublicKeyAggregator, + Addr>, + E3id, +)> { + let (bus, rng, _seed, params, crp, _errors, history) = + get_common_setup(Some(BfvPreset::InsecureThreshold512.into()))?; + let e3_id = E3id::new("42", 1); + let fhe = Arc::new(Fhe::new(params, crp, rng)); + let aggregator = PublicKeyAggregator::new( + PublicKeyAggregatorParams { + fhe, + bus, + e3_id: e3_id.clone(), + params_preset: BfvPreset::InsecureThreshold512, + committee_size, + }, + test_state(initial_state), + ); + + Ok((aggregator, history, e3_id)) +} + +fn c1_proof_with_pk_commitment(e3_id: &E3id, pk_commitment: [u8; 32]) -> SignedProofPayload { + let mut signals = vec![0u8; 96]; + signals[32..64].copy_from_slice(&pk_commitment); + SignedProofPayload { + payload: ProofPayload { + e3_id: e3_id.clone(), + proof_type: ProofType::C1PkGeneration, + proof: Proof::new( + CircuitName::PkGeneration, + ArcBytes::from_bytes(&[1]), + ArcBytes::from_bytes(&signals), + ), + }, + signature: ArcBytes::from_bytes(&[0u8; 65]), + } +} + +fn verifying_c1_non_square_state( + fhe: &Fhe, + e3_id: &E3id, +) -> Result<(PublicKeyAggregatorState, usize, usize, usize)> { + use fhe::bfv::SecretKey; + use fhe::mbfv::PublicKeyShare; + use fhe_traits::Serialize; + + let committee = CiphernodesCommitteeSize::Micro.values(); + let threshold_n = committee.n; + let threshold_m = committee.threshold; + let circuit_h = committee.h; + assert_ne!( + threshold_n, circuit_h, + "test requires a non-square committee (N != H)" + ); + + let mut submission_order = Vec::with_capacity(threshold_n); + let mut c1_proofs = Vec::with_capacity(threshold_n); + let mut rng = rand::rng(); + + for party_id in 0..threshold_n as u64 { + let node = format!("0x{:040x}", party_id + 1); + if party_id < circuit_h as u64 { + let sk = SecretKey::random(&fhe.params, &mut rng); + let pk_share = PublicKeyShare::new(&sk, fhe.crp.clone(), &mut rng)?; + let ks_bytes = ArcBytes::from_bytes(&pk_share.to_bytes()); + let commitment = e3_zk_helpers::compute_pk_commitment_from_keyshare_bytes( + &ks_bytes, + &fhe.params, + &fhe.crp, + )?; + submission_order.push((party_id, node, ks_bytes)); + c1_proofs.push(Some(c1_proof_with_pk_commitment(e3_id, commitment))); + } else { + submission_order.push((party_id, node, ArcBytes::from_bytes(&[party_id as u8]))); + c1_proofs.push(None); + } + } + + Ok(( + PublicKeyAggregatorState::VerifyingC1 { + submission_order, + threshold_m, + circuit_committee_n: threshold_n, + circuit_committee_h: circuit_h, + c1_proofs, + no_proof_parties: vec![], + }, + threshold_n, + threshold_m, + circuit_h, + )) +} + +async fn next_event(history: &Addr>) -> Result { + let mut result = history.send(TakeEvents::::new(1)).await?; + assert!(!result.timed_out, "timed out waiting for an event"); + Ok(result.events.pop().expect("expected one event")) +} + +mod attestations; +mod failures; diff --git a/crates/aggregator/src/public_key_aggregation/transitions.rs b/crates/aggregator/src/public_key_aggregation/transitions.rs new file mode 100644 index 0000000000..c73d8691f4 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/transitions.rs @@ -0,0 +1,256 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Pure public-key aggregation decisions and state transitions. + +use super::*; + +/// Decision returned by [`PublicKeyAggregation::plan_c1_dispatch`]: which parties have a C1 +/// proof to verify and which submitted a keyshare without one (treated as dishonest). +pub(crate) struct C1Dispatch { + pub party_proofs: Vec, + pub no_proof_parties: Vec, +} + +/// Outcome of [`PublicKeyAggregation::select_honest_set`]. +pub(crate) enum HonestSelection { + /// Too few honest parties cleared C1 — caller must fail the E3. + Fail, + /// Enough honest parties — proceed to aggregation with this honest set. + Proceed { + honest_entries: Vec<(u64, String, ArcBytes, Option)>, + honest_party_ids: BTreeSet, + }, +} + +/// Plain, synchronous domain service for public-key aggregation decisions. +pub(crate) struct PublicKeyAggregation; + +impl PublicKeyAggregation { + /// Add a keyshare to a `Collecting` state, returning the next state. When all `N` + /// committee keyshares have arrived this transitions to `VerifyingC1`. Submitting a + /// `party_id` that already has a keyshare is idempotent (state is returned unchanged). + pub(crate) fn add_keyshare( + mut state: PublicKeyAggregatorState, + keyshare: ArcBytes, + node: String, + party_id: u64, + c1_proof: Option, + ) -> Result { + let PublicKeyAggregatorState::Collecting { + threshold_n, + threshold_m, + circuit_committee_n, + circuit_committee_h, + keyshares, + c1_proofs, + nodes, + submission_order, + .. + } = &mut state + else { + return Err(anyhow::anyhow!("Can only add keyshare in Collecting state")); + }; + + if submission_order.iter().any(|(pid, _, _)| *pid == party_id) { + return Ok(state); + } + + keyshares.insert(keyshare.clone()); + c1_proofs.push(c1_proof); + nodes.insert(node.clone()); + info!( + "add_keyshare: node={node} party_id={party_id} (arrival slot={})", + submission_order.len() + ); + submission_order.push((party_id, node, keyshare)); + let n = *threshold_n; + let m = *threshold_m; + let committee_n = *circuit_committee_n; + let committee_h = *circuit_committee_h; + let unique_parties = submission_order.len(); + info!( + "PublicKeyAggregator got keyshares {unique_parties}/{n} distinct parties (circuit_n={committee_n}, committee_h={committee_h})" + ); + // Collect all N committee keyshares before C1. C5 then requires exactly H honest + // proofs afterward (micro had N=H so waiting for H was equivalent). + if unique_parties >= n { + info!( + "Collected keyshares from {unique_parties} distinct parties (>= live_n={n}, circuit_n={committee_n}), transitioning to VerifyingC1..." + ); + return Ok(PublicKeyAggregatorState::VerifyingC1 { + submission_order: std::mem::take(submission_order), + threshold_m: m, + circuit_committee_n: committee_n, + circuit_committee_h: committee_h, + c1_proofs: std::mem::take(c1_proofs), + no_proof_parties: Vec::new(), + }); + } + + Ok(state) + } + + /// Split the collected keyshare submissions into parties with a C1 proof to verify and + /// parties that submitted no proof (treated as dishonest by the caller). + pub(crate) fn plan_c1_dispatch( + submission_order: &[(u64, String, ArcBytes)], + c1_proofs: &[Option], + ) -> C1Dispatch { + let mut party_proofs = Vec::new(); + let mut no_proof_parties = Vec::new(); + + for ((party_id, _, _), proof_opt) in submission_order.iter().zip(c1_proofs.iter()) { + match proof_opt { + Some(proof) => { + party_proofs.push(PartyProofsToVerify { + sender_party_id: *party_id, + signed_proofs: vec![proof.clone()], + }); + } + None => { + warn!( + "Party {} submitted keyshare without C1 proof — treating as dishonest", + party_id + ); + no_proof_parties.push(*party_id); + } + } + } + + C1Dispatch { + party_proofs, + no_proof_parties, + } + } + + /// Select the canonical honest set after C1 (ZK + commitment) filtering. + /// + /// Sorts the honest entries by real `party_id`, fails closed when fewer than `circuit_h` + /// parties cleared C1, caps the honest set to the `circuit_h` lowest `party_id`s, and + /// fails again if `<= threshold_m` parties remain. Logging mirrors the original handler. + pub(crate) fn select_honest_set( + e3_id: &E3id, + mut honest_entries: Vec<(u64, String, ArcBytes, Option)>, + dishonest_parties: &BTreeSet, + circuit_h: usize, + threshold_m: usize, + collected: usize, + ) -> HonestSelection { + // Sort by real party_id ascending so honest_keyshares / honest_nodes / + // honest_party_ids all share the same ordering used by NodeFold rows + // and by the circuit's slot indexing in `dkg_aggregator.nr`. + honest_entries.sort_by_key(|(pid, _, _, _)| *pid); + + if !dishonest_parties.is_empty() { + warn!( + "Total dishonest parties (ZK + commitment): {:?}", + dishonest_parties + ); + } + + // Fail closed when fewer than H parties cleared C1 — C5 cannot be witnessed. + if honest_entries.len() < circuit_h { + error!( + "C5 requires {circuit_h} honest parties with valid C1 proofs; only {} honest after verification (collected {collected}, dishonest: {:?})", + honest_entries.len(), + dishonest_parties + ); + return HonestSelection::Fail; + } + + // The C5 PkAggregation circuit is parameterised by a fixed honest-party count H. + // When more than H parties cleared C1, select the H lowest party_ids as the + // canonical honest set; the remainder stay in the full committee. + let pre_cap_len = honest_entries.len(); + let honest_party_ids = + cap_honest_party_ids(circuit_h, honest_entries.iter().map(|(pid, _, _, _)| *pid)); + if pre_cap_len > circuit_h { + info!( + "Capping honest set from {pre_cap_len} to circuit_h={circuit_h} for E3 {e3_id} (extras remain in full committee)" + ); + honest_entries.retain(|(pid, _, _, _)| honest_party_ids.contains(pid)); + } + + // Defensive: should hold after truncation above; guard against future refactors. + if honest_entries.len() <= threshold_m { + error!( + "Not enough honest parties after filtering: {} (need > {})", + honest_entries.len(), + threshold_m + ); + return HonestSelection::Fail; + } + + HonestSelection::Proceed { + honest_entries, + honest_party_ids, + } + } + + /// Apply a committee-member expulsion to a `Collecting` state, keeping the parallel + /// collections aligned and transitioning to `VerifyingC1` when enough keyshares remain. + pub(crate) fn handle_member_expelled( + mut state: PublicKeyAggregatorState, + node: &str, + ) -> Result { + let PublicKeyAggregatorState::Collecting { + threshold_n, + threshold_m, + circuit_committee_n, + circuit_committee_h, + keyshares, + c1_proofs, + nodes, + submission_order, + .. + } = &mut state + else { + return Ok(state); + }; + + let node_str = node.to_string(); + + // Find the expelled node's index in submission_order and remove from + // all parallel collections so they stay aligned. + if let Some(idx) = submission_order.iter().position(|(_, n, _)| n == &node_str) { + let (_, _, expelled_keyshare) = submission_order.remove(idx); + keyshares.remove(&expelled_keyshare); + c1_proofs.remove(idx); + } + + nodes.remove(&node_str); + + if *threshold_n > 0 { + *threshold_n -= 1; + info!( + "PublicKeyAggregator: reduced threshold_n to {} after expelling {}", + threshold_n, node + ); + } + + if *threshold_n < *threshold_m { + warn!( + "PublicKeyAggregator: threshold_n ({}) < threshold_m ({}) after expulsion — committee unviable", + threshold_n, threshold_m + ); + return Ok(state); + } + + if keyshares.len() == *threshold_n && *threshold_n > 0 { + let m = *threshold_m; + let committee_n = *circuit_committee_n; + let committee_h = *circuit_committee_h; + info!("PublicKeyAggregator: enough keyshares after expulsion, transitioning to VerifyingC1"); + return Ok(PublicKeyAggregatorState::VerifyingC1 { + submission_order: std::mem::take(submission_order), + threshold_m: m, + circuit_committee_n: committee_n, + circuit_committee_h: committee_h, + c1_proofs: std::mem::take(c1_proofs), + no_proof_parties: Vec::new(), + }); + } + + Ok(state) + } +} diff --git a/crates/aggregator/src/public_key_aggregation/validation.rs b/crates/aggregator/src/public_key_aggregation/validation.rs new file mode 100644 index 0000000000..c585a93615 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/validation.rs @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Cryptographic commitment and DKG-fold attestation checks. + +use super::*; + +/// Circuit honest-party count `H` for the committee `(threshold_m, threshold_n)`. +pub(crate) fn committee_h_for(threshold_m: usize, threshold_n: usize) -> Result { + Ok( + CiphernodesCommitteeSize::from_threshold(threshold_m, threshold_n) + .with_context(|| { + format!("unknown committee for threshold_m={threshold_m} threshold_n={threshold_n}") + })? + .values() + .h, + ) +} + +/// Public-signal key for the aggregated PK commitment in `CircuitName::PkAggregation` (C5). +/// Must stay in lock-step with the Noir circuit's output ABI declaration. +const C5_PK_COMMITMENT_FIELD: &str = "commitment"; + +#[allow(clippy::too_many_arguments)] +pub(crate) fn verify_dkg_fold_attestation( + e3_id: &E3id, + party_id: u64, + proof: &Proof, + attestation: &SignedDkgFoldAttestation, + expected_node: &str, + committee_n: usize, + committee_h: usize, + n_moduli: usize, +) -> Result<()> { + ensure!( + attestation.payload.e3_id == *e3_id, + "attestation e3_id mismatch" + ); + ensure!( + attestation.payload.party_id == party_id, + "attestation party_id mismatch" + ); + let expected: Address = expected_node + .parse() + .with_context(|| format!("invalid committee node address {expected_node}"))?; + ensure!( + attestation.verify_signer(&expected)?, + "fold attestation signer does not match committee node for party {party_id}" + ); + let (extracted_party, commits) = + extract_node_fold_agg_commits(proof, committee_n, committee_h, n_moduli) + .map_err(|e| anyhow!("{e}"))?; + ensure!(extracted_party == party_id, "NodeFold party_id mismatch"); + ensure!( + commits == attestation.payload.agg_commits, + "NodeFold commits do not match signed attestation" + ); + Ok(()) +} + +/// Extract the hash-based aggregated PK commitment from the signed C5 proof. +/// This is the last public signal of `CircuitName::PkAggregation`. +pub(crate) fn extract_pk_commitment(c5_proof: &Proof) -> Result<[u8; 32]> { + let layout = CircuitName::PkAggregation.output_layout(); + let bytes = layout + .extract_field(&c5_proof.public_signals, C5_PK_COMMITMENT_FIELD) + .ok_or_else(|| anyhow::anyhow!("C5 proof is missing `commitment` public signal"))?; + let mut out = [0u8; 32]; + if bytes.len() != 32 { + return Err(anyhow::anyhow!( + "C5 `commitment` public signal must be 32 bytes" + )); + } + out.copy_from_slice(bytes); + Ok(out) +} + +/// Outcome of cross-checking each honest party's keyshare against its signed C1 +/// `pk_commitment` public signal. +pub(crate) struct C1CommitmentAudit { + /// Parties whose keyshare does not recompute to their signed C1 commitment, paired + /// with the proof for `SignedProofFailed` reporting. + pub mismatched: Vec<(u64, SignedProofPayload)>, + /// Parties that carried no C1 proof at all (defensive — normally already dishonest). + pub missing_proof: Vec, +} + +/// Recompute each honest party's `pk_commitment` from its keyshare bytes and compare it +/// against the `pk_commitment` public signal in the party's signed C1 proof. Pure: the +/// actor publishes `SignedProofFailed` for `mismatched` and folds both result sets into +/// `dishonest_parties`. +pub(crate) fn check_c1_keyshare_commitments( + entries: &[(u64, String, ArcBytes, Option)], + fhe: &Fhe, +) -> C1CommitmentAudit { + let mut mismatched = Vec::new(); + let mut missing_proof = Vec::new(); + for (party_id, _node, ks, c1) in entries { + let Some(signed_proof) = c1.as_ref() else { + warn!( + "Party {} has no C1 proof but was not marked dishonest", + party_id + ); + missing_proof.push(*party_id); + continue; + }; + let ok = match e3_zk_helpers::compute_pk_commitment_from_keyshare_bytes( + ks, + &fhe.params, + &fhe.crp, + ) { + Ok(computed) => signed_proof + .payload + .proof + .extract_output("pk_commitment") + .is_some_and(|extracted| extracted[..] == computed[..]), + Err(e) => { + warn!( + "Failed to compute pk_commitment for party {}: {}", + party_id, e + ); + false + } + }; + if !ok { + mismatched.push((*party_id, signed_proof.clone())); + } + } + C1CommitmentAudit { + mismatched, + missing_proof, + } +} diff --git a/crates/aggregator/src/public_key_aggregation/workflow.rs b/crates/aggregator/src/public_key_aggregation/workflow.rs new file mode 100644 index 0000000000..2aa6ace06f --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/workflow.rs @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Deterministic public-key aggregation workflow. +//! +//! This module holds the [`PublicKeyAggregatorState`] state machine plus the pure +//! transition/decision functions used by the `PublicKeyAggregator` actor. Nothing here +//! touches actix, `Persistable`, or the event bus: the actor feeds inputs in, gets a +//! next-state or a [decision](C1Dispatch)/[`HonestSelection`] back, and performs the +//! persistence/publish/dispatch side effects itself. + +use alloy::primitives::Address; +use anyhow::{anyhow, ensure, Context as _, Result}; +use e3_events::{ + CircuitName, E3id, OrderedSet, PartyProofsToVerify, Proof, Seed, SignedDkgFoldAttestation, + SignedProofPayload, +}; +use e3_fhe::Fhe; +use e3_utils::ArcBytes; +use e3_zk_helpers::cap_honest_party_ids; +use e3_zk_helpers::CiphernodesCommitteeSize; +use e3_zk_prover::extract_node_fold_agg_commits; +use std::collections::{BTreeSet, HashMap}; +use tracing::{error, info, warn}; + +#[path = "state.rs"] +mod state; +#[path = "transitions.rs"] +mod transitions; +#[path = "validation.rs"] +mod validation; + +pub use state::PublicKeyAggregatorState; +pub(crate) use transitions::{C1Dispatch, HonestSelection, PublicKeyAggregation}; +pub(crate) use validation::{ + check_c1_keyshare_commitments, committee_h_for, extract_pk_commitment, + verify_dkg_fold_attestation, +}; + +#[cfg(test)] +#[path = "workflow_tests.rs"] +mod tests; diff --git a/crates/aggregator/src/public_key_aggregation/workflow_tests.rs b/crates/aggregator/src/public_key_aggregation/workflow_tests.rs new file mode 100644 index 0000000000..5789834596 --- /dev/null +++ b/crates/aggregator/src/public_key_aggregation/workflow_tests.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +fn ks(byte: u8) -> ArcBytes { + ArcBytes::from_bytes(&[byte]) +} + +fn collecting(threshold_n: usize, threshold_m: usize) -> PublicKeyAggregatorState { + PublicKeyAggregatorState::init(threshold_n, threshold_m, Seed([0u8; 32])) +} + +#[test] +fn add_keyshare_below_threshold_stays_collecting() { + // minimum committee maps (m=1, n=3) -> needs 3 parties. + let state = collecting(3, 1); + let next = + PublicKeyAggregation::add_keyshare(state, ks(1), "node-0".into(), 0, None).expect("add ok"); + match next { + PublicKeyAggregatorState::Collecting { + submission_order, .. + } => assert_eq!(submission_order.len(), 1), + _ => panic!("expected Collecting"), + } +} + +#[test] +fn add_keyshare_duplicate_party_is_idempotent() { + let state = collecting(3, 1); + let state = PublicKeyAggregation::add_keyshare(state, ks(1), "node-0".into(), 0, None).unwrap(); + let state = PublicKeyAggregation::add_keyshare(state, ks(9), "node-0".into(), 0, None).unwrap(); + match state { + PublicKeyAggregatorState::Collecting { + submission_order, + keyshares, + .. + } => { + assert_eq!(submission_order.len(), 1, "duplicate party ignored"); + assert_eq!(keyshares.len(), 1); + } + _ => panic!("expected Collecting"), + } +} + +#[test] +fn add_keyshare_reaching_threshold_transitions_to_verifying_c1() { + let mut state = collecting(3, 1); + for pid in 0..3u64 { + state = PublicKeyAggregation::add_keyshare( + state, + ks(pid as u8), + format!("node-{pid}"), + pid, + None, + ) + .unwrap(); + } + match state { + PublicKeyAggregatorState::VerifyingC1 { + submission_order, + circuit_committee_n, + circuit_committee_h, + threshold_m, + .. + } => { + assert_eq!(submission_order.len(), 3); + assert_eq!(circuit_committee_n, 3); + assert_eq!(circuit_committee_h, 2); + assert_eq!(threshold_m, 1); + } + _ => panic!("expected VerifyingC1"), + } +} + +#[test] +fn add_keyshare_wrong_state_errors() { + let state = PublicKeyAggregatorState::VerifyingC1 { + submission_order: vec![], + threshold_m: 1, + circuit_committee_n: 3, + circuit_committee_h: 2, + c1_proofs: vec![], + no_proof_parties: vec![], + }; + let err = PublicKeyAggregation::add_keyshare(state, ks(1), "n".into(), 0, None); + assert!(err.is_err()); +} + +#[test] +fn plan_c1_dispatch_splits_proofs_and_missing() { + let submission_order = vec![ + (0u64, "node-0".to_string(), ks(1)), + (1u64, "node-1".to_string(), ks(2)), + ]; + // party 0 has no proof, party 1 has one + let c1_proofs: Vec> = vec![None, None]; + let plan = PublicKeyAggregation::plan_c1_dispatch(&submission_order, &c1_proofs); + // both None -> both treated as no-proof + assert_eq!(plan.no_proof_parties, vec![0, 1]); + assert!(plan.party_proofs.is_empty()); +} + +#[test] +fn select_honest_set_fails_below_circuit_h() { + let e3_id = E3id::new("1", 1); + let honest = vec![(0u64, "n0".to_string(), ks(1), None)]; + let sel = PublicKeyAggregation::select_honest_set(&e3_id, honest, &BTreeSet::new(), 3, 1, 1); + assert!(matches!(sel, HonestSelection::Fail)); +} + +#[test] +fn select_honest_set_caps_to_circuit_h_and_sorts() { + let e3_id = E3id::new("1", 1); + // 4 honest, circuit_h = 3 -> cap to lowest-3 party_ids {0,1,2}, sorted ascending. + let honest = vec![ + (3u64, "n3".to_string(), ks(4), None), + (1u64, "n1".to_string(), ks(2), None), + (0u64, "n0".to_string(), ks(1), None), + (2u64, "n2".to_string(), ks(3), None), + ]; + let sel = PublicKeyAggregation::select_honest_set(&e3_id, honest, &BTreeSet::new(), 3, 1, 4); + match sel { + HonestSelection::Proceed { + honest_entries, + honest_party_ids, + } => { + let ids: Vec = honest_entries.iter().map(|(p, _, _, _)| *p).collect(); + assert_eq!(ids, vec![0, 1, 2]); + assert_eq!(honest_party_ids, BTreeSet::from([0, 1, 2])); + } + HonestSelection::Fail => panic!("expected Proceed"), + } +} + +#[test] +fn select_honest_set_fails_when_at_or_below_threshold_m() { + let e3_id = E3id::new("1", 1); + // 3 honest, circuit_h = 3 but threshold_m = 3 -> len <= m -> Fail. + let honest = vec![ + (0u64, "n0".to_string(), ks(1), None), + (1u64, "n1".to_string(), ks(2), None), + (2u64, "n2".to_string(), ks(3), None), + ]; + let sel = PublicKeyAggregation::select_honest_set(&e3_id, honest, &BTreeSet::new(), 3, 3, 3); + assert!(matches!(sel, HonestSelection::Fail)); +} + +#[test] +fn handle_member_expelled_removes_and_reduces_threshold() { + let mut state = collecting(3, 1); + for pid in 0..2u64 { + state = PublicKeyAggregation::add_keyshare( + state, + ks(pid as u8), + format!("node-{pid}"), + pid, + None, + ) + .unwrap(); + } + // expel node-0; threshold_n 3 -> 2, keyshares now 1 (< 2) -> stays Collecting + let state = PublicKeyAggregation::handle_member_expelled(state, "node-0").unwrap(); + match state { + PublicKeyAggregatorState::Collecting { + threshold_n, + submission_order, + nodes, + .. + } => { + assert_eq!(threshold_n, 2); + assert_eq!(submission_order.len(), 1); + assert!(!nodes.contains(&"node-0".to_string())); + } + _ => panic!("expected Collecting"), + } +} + +#[test] +fn handle_member_expelled_transitions_when_enough_remain() { + // Collecting with n=2, m=1, two keyshares present; expel one -> + // threshold_n 1, keyshares 1 == n -> VerifyingC1. + let state = PublicKeyAggregatorState::Collecting { + threshold_n: 2, + threshold_m: 1, + circuit_committee_n: 3, + circuit_committee_h: 2, + keyshares: OrderedSet::from(vec![ks(10), ks(11)]), + c1_proofs: vec![None, None], + seed: Seed([0u8; 32]), + nodes: OrderedSet::from(vec!["node-0".to_string(), "node-1".to_string()]), + submission_order: vec![ + (0, "node-0".to_string(), ks(10)), + (1, "node-1".to_string(), ks(11)), + ], + }; + let next = PublicKeyAggregation::handle_member_expelled(state, "node-0").unwrap(); + match next { + PublicKeyAggregatorState::VerifyingC1 { + circuit_committee_n, + circuit_committee_h, + submission_order, + .. + } => { + assert_eq!(circuit_committee_n, 3); + assert_eq!(circuit_committee_h, 2); + assert_eq!(submission_order.len(), 1); + } + _ => panic!("expected VerifyingC1"), + } +} diff --git a/crates/aggregator/src/workflow.rs b/crates/aggregator/src/workflow.rs new file mode 100644 index 0000000000..2de888148a --- /dev/null +++ b/crates/aggregator/src/workflow.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Compatibility view of aggregation workflows stored by capability. +//! +//! These modules own protocol progress but perform no Actix, persistence, or bus I/O. The actor +//! shells commit their state transitions and execute the resulting effects. + +#[path = "public_key_aggregation/workflow.rs"] +pub mod publickey_aggregation; +#[path = "plaintext_aggregation/workflow.rs"] +pub mod threshold_plaintext_aggregation; diff --git a/crates/data/src/persistable.rs b/crates/data/src/persistable.rs index af76cccbb1..0035104659 100644 --- a/crates/data/src/persistable.rs +++ b/crates/data/src/persistable.rs @@ -166,30 +166,38 @@ where Ok(Some(e3_utils::deserialize_exact(&bytes)?)) } - fn write_to_store(&self) { + fn write_value_to_store(&self, data: &T) -> Result<()> { if self.staging_mode { - return; + return Ok(()); } - let Some(ref data) = self.data else { - return; - }; - let Result::Ok(serialized) = bincode::serialize(data) else { - tracing::error!("Could not serialize value for persistable"); - return; - }; + let serialized = + bincode::serialize(data).context("could not serialize value for persistable")?; let msg = if let Some(ctx) = self.ctx.clone() { Insert::new_with_context(&self.connector.key, serialized, ctx) } else { Insert::new(&self.connector.key, serialized) }; - self.connector.insert.do_send(msg); + self.connector + .insert + .try_send(msg) + .context("persistable store mailbox rejected snapshot write")?; + Ok(()) + } + + fn write_to_store(&self) -> Result<()> { + let Some(ref data) = self.data else { + return Ok(()); + }; + self.write_value_to_store(data) } /// Save the data in the container to the store pub fn save(self) -> Self { - self.write_to_store(); + if let Err(error) = self.write_to_store() { + tracing::error!(%error, "Could not enqueue persistable snapshot"); + } self } @@ -212,25 +220,43 @@ where where F: FnOnce(T) -> Result, { - self.ctx = ctx; // Set the context + self.ctx = ctx; let content = self.data.clone().ok_or(anyhow!("Data has not been set"))?; - self.data = Some(mutator(content)?); - self.write_to_store(); + let next = mutator(content)?; + // Accept the snapshot write before exposing the new state in memory. + // The append-only event log remains the durable source of truth; this + // ordering prevents a saturated store mailbox from silently advancing + // only the actor-local snapshot. + self.write_value_to_store(&next)?; + self.data = Some(next); Ok(()) } /// Set the data on both the persistable and the store pub fn set(&mut self, data: T) { + if self.staging_mode { + self.data = Some(data); + return; + } + if let Err(error) = self.write_value_to_store(&data) { + tracing::error!(%error, "Could not enqueue persistable snapshot"); + return; + } self.data = Some(data); - self.write_to_store(); } /// Clear the data from both the persistable and the store pub fn clear(&mut self) { - self.data = None; - self.connector + match self + .connector .remove - .do_send(Remove::new(&self.connector.key)); + .try_send(Remove::new(&self.connector.key)) + { + std::result::Result::Ok(()) => self.data = None, + std::result::Result::Err(error) => { + tracing::error!(%error, "Could not enqueue persistable snapshot removal") + } + } } /// Get the data currently stored on the container as an Option @@ -258,7 +284,10 @@ where /// Commit mode - writes current state and enables persistence pub fn commit(&mut self) { self.staging_mode = false; - self.write_to_store(); + if let Err(error) = self.write_to_store() { + self.staging_mode = true; + tracing::error!(%error, "Could not enqueue staged persistable snapshot"); + } } } @@ -300,6 +329,10 @@ mod tests { #[rtype("Vec")] struct GetEvents; + #[derive(Message)] + #[rtype(result = "()")] + struct Stop; + impl Actor for MockConnector { type Context = actix::Context; fn started(&mut self, ctx: &mut Self::Context) { @@ -314,6 +347,15 @@ mod tests { } } + impl Handler for MockConnector { + type Result = (); + + fn handle(&mut self, _: Stop, ctx: &mut Self::Context) -> Self::Result { + use actix::ActorContext as _; + ctx.stop(); + } + } + impl Handler for MockConnector { type Result = Option>; fn handle(&mut self, _msg: Get, _ctx: &mut Self::Context) -> Self::Result { @@ -384,4 +426,20 @@ mod tests { matches!(&events[1], Evts::Insert(msg) if msg.value() == &bincode::serialize(&200i32).unwrap()) ); } + + #[actix::test] + async fn rejected_snapshot_write_does_not_advance_in_memory_state() { + let (addr, connector) = MockConnector::new(b"loc").to_store_connector(); + let mut persistable = Persistable::new(Some(42i32), connector); + + addr.send(Stop).await.unwrap(); + actix::clock::sleep(std::time::Duration::from_millis(10)).await; + + let error = persistable + .try_mutate_without_context(|value| Ok(value + 1)) + .unwrap_err(); + + assert!(error.to_string().contains("mailbox rejected")); + assert_eq!(persistable.get(), Some(42)); + } } diff --git a/crates/evm/src/actors/mod.rs b/crates/evm/src/actors.rs similarity index 68% rename from crates/evm/src/actors/mod.rs rename to crates/evm/src/actors.rs index d8cbd8c3e9..270a7bb184 100644 --- a/crates/evm/src/actors/mod.rs +++ b/crates/evm/src/actors.rs @@ -4,22 +4,33 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -//! Thin actix message-passing shells wrapping the pure services in `domain` -//! and performing the EVM/provider I/O (transport). +//! Compatibility view of thin actors stored with their EVM capabilities. +#[path = "bonding_registry/actor.rs"] mod bonding_registry_sol; +#[path = "ciphernode_registry/actor.rs"] mod ciphernode_registry_sol; +#[path = "chain_gateway/actor.rs"] mod evm_chain_gateway; +#[path = "chain_hub.rs"] mod evm_hub; +#[path = "event_decoding/actor.rs"] mod evm_parser; +#[path = "chain_reader/actor.rs"] mod evm_read_interface; +#[path = "event_router.rs"] mod evm_router; +#[path = "historical_order/actor.rs"] mod fix_historical_order; +#[path = "interfold/reader.rs"] mod interfold_sol_reader; +#[path = "interfold_writing/actor.rs"] mod interfold_sol_writer; -mod log_fetcher; +#[path = "slashing/reader.rs"] mod slashing_manager_sol_reader; +#[path = "slashing_writing/actor.rs"] mod slashing_manager_sol_writer; +#[path = "chain_sync/start_extractor.rs"] mod sync_start_extractor; pub use bonding_registry_sol::BondingRegistrySolReader; diff --git a/crates/evm/src/actors/ciphernode_registry_sol.rs b/crates/evm/src/actors/ciphernode_registry_sol.rs deleted file mode 100644 index c09cd925a0..0000000000 --- a/crates/evm/src/actors/ciphernode_registry_sol.rs +++ /dev/null @@ -1,590 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::actors::evm_parser::EvmParser; -use crate::contracts::ICiphernodeRegistry; -use crate::domain::ciphernode_registry_events::extractor; -use crate::domain::error_decoder::{decode_error_from_str, format_evm_error}; -use crate::helpers::{encode_zk_proof, send_tx_with_retry, transaction_nonce_guard, EthProvider}; -use crate::messages::{EvmEventProcessor, InterfoldEvmEvent}; -use actix::prelude::*; -use alloy::{ - primitives::{Address, Bytes, B256, U256}, - providers::{Provider, WalletProvider}, - rpc::types::TransactionReceipt, -}; -use anyhow::Result; -use e3_events::{ - prelude::*, AggregatorChanged, BusHandle, CommitteeFinalizeRequested, E3RequestComplete, E3id, - EType, EffectsEnabled, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, Proof, - PublicKeyAggregated, Shutdown, TicketGenerated, TicketId, -}; -use e3_utils::{require_successful_receipt, ArcBytes, NotifySync, MAILBOX_LIMIT}; -use std::collections::{HashMap, HashSet}; -use tracing::{error, info}; - -/// Connects to CiphernodeRegistry.sol converting EVM events to InterfoldEvents -pub struct CiphernodeRegistrySolReader; - -impl CiphernodeRegistrySolReader { - pub fn setup(next: &EvmEventProcessor) -> Addr { - EvmParser::new(next, extractor).start() - } -} - -/// Writer for publishing committees to CiphernodeRegistry -pub struct CiphernodeRegistrySolWriter

{ - provider: EthProvider

, - contract_address: Address, - bus: BusHandle, - effects_enabled: bool, - active_aggregators: HashMap, - /// E3s for which a `publishCommittee` submission is currently in flight. - /// Guards against firing a second on-chain tx for the same E3 before the - /// first is observed (wasted gas / nonce contention, H13). The authoritative - /// cross-restart idempotency remains the on-chain `should_publish_committee` - /// preflight; this only dedups concurrent in-session submissions. Entries - /// are cleared on failure so a genuine retry can proceed. - submitting: HashSet, -} - -impl CiphernodeRegistrySolWriter

{ - pub fn new( - bus: &BusHandle, - provider: EthProvider

, - contract_address: Address, - ) -> Result { - Ok(Self { - provider, - contract_address, - bus: bus.clone(), - effects_enabled: false, - active_aggregators: HashMap::new(), - submitting: HashSet::new(), - }) - } - - pub fn attach(bus: &BusHandle, provider: EthProvider

, contract_address: Address) { - let addr = CiphernodeRegistrySolWriter::new(bus, provider, contract_address) - .expect("failed to create CiphernodeRegistrySolWriter") - .start(); - - bus.subscribe_all( - &[ - EventType::EffectsEnabled, - EventType::AggregatorChanged, - EventType::PublicKeyAggregated, - EventType::CommitteeFinalizeRequested, - EventType::TicketGenerated, - EventType::E3RequestComplete, - EventType::Shutdown, - ], - addr.into(), - ); - } - - fn is_active_aggregator_for(&self, e3_id: &E3id) -> bool { - self.active_aggregators.get(e3_id).copied().unwrap_or(false) - } -} - -impl Actor for CiphernodeRegistrySolWriter

{ - type Context = actix::Context; - - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT) - } -} - -impl Handler - for CiphernodeRegistrySolWriter

-{ - type Result = (); - - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - match msg.into_data() { - InterfoldEventData::EffectsEnabled(data) => self.notify_sync(ctx, data), - InterfoldEventData::AggregatorChanged(data) => self.notify_sync(ctx, data), - InterfoldEventData::PublicKeyAggregated(data) => { - // Only publish if the src and destination chains match - if self.provider.chain_id() == data.e3_id.chain_id() { - ctx.notify(data); - } - } - InterfoldEventData::CommitteeFinalizeRequested(data) => { - if self.provider.chain_id() == data.e3_id.chain_id() { - ctx.notify(data); - } - } - InterfoldEventData::TicketGenerated(data) => { - // Submit ticket if chain matches - if self.provider.chain_id() == data.e3_id.chain_id() { - ctx.notify(data); - } - } - InterfoldEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), - InterfoldEventData::Shutdown(data) => self.notify_sync(ctx, data), - _ => (), - } - } -} - -impl Handler - for CiphernodeRegistrySolWriter

-{ - type Result = (); - - fn handle(&mut self, _: EffectsEnabled, _: &mut Self::Context) -> Self::Result { - self.effects_enabled = true; - } -} - -impl Handler - for CiphernodeRegistrySolWriter

-{ - type Result = (); - - fn handle(&mut self, msg: AggregatorChanged, _: &mut Self::Context) -> Self::Result { - self.active_aggregators.insert(msg.e3_id, msg.is_aggregator); - } -} - -impl Handler - for CiphernodeRegistrySolWriter

-{ - type Result = (); - - fn handle(&mut self, msg: E3RequestComplete, _: &mut Self::Context) -> Self::Result { - self.active_aggregators.remove(&msg.e3_id); - self.submitting.remove(&msg.e3_id); - } -} - -impl Handler - for CiphernodeRegistrySolWriter

-{ - type Result = ResponseFuture<()>; - - fn handle(&mut self, msg: TicketGenerated, _: &mut Self::Context) -> Self::Result { - if !self.effects_enabled { - return Box::pin(async {}); - } - - match msg.ticket_id { - TicketId::Score(ticket_id) => { - info!( - "Score sortition ticket generated for E3 {:?}, submitting to contract", - msg.e3_id - ); - - let e3_id = msg.e3_id.clone(); - let contract_address = self.contract_address; - let provider = self.provider.clone(); - let bus = self.bus.clone(); - - Box::pin(async move { - info!("Submitting ticket {} for E3 {:?}", ticket_id, e3_id); - - let result = - submit_ticket_to_registry(provider, contract_address, e3_id, ticket_id) - .await; - match result { - Ok(receipt) => { - info!(tx=%receipt.transaction_hash, "Ticket submitted to registry"); - } - Err(err) => { - error!("Failed to submit ticket: {}", format_evm_error(&err)); - bus.err(EType::Evm, err); - } - } - }) - } - } - } -} - -impl Handler - for CiphernodeRegistrySolWriter

-{ - type Result = ResponseFuture<()>; - - fn handle(&mut self, msg: CommitteeFinalizeRequested, _: &mut Self::Context) -> Self::Result { - if !self.effects_enabled { - return Box::pin(async {}); - } - - let e3_id = msg.e3_id.clone(); - let contract_address = self.contract_address; - let provider = self.provider.clone(); - let bus = self.bus.clone(); - - Box::pin(async move { - match should_finalize_committee(provider.clone(), contract_address, e3_id.clone()).await - { - Ok(false) => { - info!(e3_id = %e3_id, "Skipping finalizeCommittee; on-chain state is not finalizable"); - return; - } - Err(err) => { - error!( - "Failed to preflight finalizeCommittee: {}", - format_evm_error(&err) - ); - return; - } - Ok(true) => {} - } - - info!("Finalizing committee for E3 {:?}", e3_id); - - let result = finalize_committee_on_registry(provider, contract_address, e3_id).await; - match result { - Ok(receipt) => { - info!(tx=%receipt.transaction_hash, "Committee finalized on registry"); - } - Err(err) => { - error!("Failed to finalize committee: {}", format_evm_error(&err)); - bus.err(EType::Evm, err); - } - } - }) - } -} - -impl Handler - for CiphernodeRegistrySolWriter

-{ - type Result = ResponseFuture<()>; - - fn handle(&mut self, msg: PublicKeyAggregated, ctx: &mut Self::Context) -> Self::Result { - if !self.effects_enabled || !self.is_active_aggregator_for(&msg.e3_id) { - return Box::pin(async {}); - } - - // Don't fire a second on-chain submission for an E3 whose publishCommittee - // tx is already in flight (H13). The on-chain preflight below is still the - // authoritative idempotency guard across restarts. - if !self.submitting.insert(msg.e3_id.clone()) { - info!(e3_id = %msg.e3_id, "publishCommittee already in flight; skipping duplicate submission"); - return Box::pin(async {}); - } - - let e3_id = msg.e3_id.clone(); - let pubkey = msg.pubkey.clone(); - let pk_commitment = msg.pk_commitment; - let dkg_aggregator_proof = msg.dkg_aggregator_proof.clone(); - let dkg_attestation_bundle = msg.dkg_attestation_bundle.clone(); - let contract_address = self.contract_address; - let provider = self.provider.clone(); - let bus = self.bus.clone(); - let self_addr = ctx.address(); - - Box::pin(async move { - match should_publish_committee(provider.clone(), contract_address, e3_id.clone()).await - { - Ok(false) => { - info!(e3_id = %e3_id, "Skipping publishCommittee; committee public key already published"); - return; - } - Err(err) => { - error!( - "Failed to preflight publishCommittee: {}", - format_evm_error(&err) - ); - // Transient read failure: allow a later event to retry. - self_addr.do_send(ClearSubmitting(e3_id)); - return; - } - Ok(true) => {} - } - - let result = publish_committee_to_registry( - provider, - contract_address, - e3_id.clone(), - pubkey, - pk_commitment, - dkg_aggregator_proof.as_ref(), - dkg_attestation_bundle.as_ref().map(|b| b.as_ref()), - ) - .await; - match result { - Ok(receipt) => { - info!(tx=%receipt.transaction_hash, "Committee published to registry"); - } - Err(err) => { - error!("Failed to publish committee: {}", format_evm_error(&err)); - // Submission failed: clear the in-flight marker so a retry can proceed. - self_addr.do_send(ClearSubmitting(e3_id)); - bus.err(EType::Evm, err); - } - } - }) - } -} - -/// Internal message: clear the in-flight `publishCommittee` marker for an E3 so -/// a subsequent submission attempt is allowed after a failure (H13). -#[derive(Message)] -#[rtype(result = "()")] -struct ClearSubmitting(E3id); - -impl Handler - for CiphernodeRegistrySolWriter

-{ - type Result = (); - - fn handle(&mut self, msg: ClearSubmitting, _: &mut Self::Context) -> Self::Result { - self.submitting.remove(&msg.0); - } -} - -impl Handler - for CiphernodeRegistrySolWriter

-{ - type Result = (); - - fn handle(&mut self, _: Shutdown, ctx: &mut Self::Context) -> Self::Result { - ctx.stop(); - } -} - -pub async fn submit_ticket_to_registry( - provider: EthProvider

, - contract_address: Address, - e3_id: E3id, - ticket_number: u64, -) -> Result { - let e3_id_u256: U256 = e3_id.try_into()?; - let ticket_number_u256 = U256::from(ticket_number); - - send_tx_with_retry("submitTicket", &["CommitteeNotRequested"], || { - let provider = provider.clone(); - async move { - info!("Calling: contract.submitTicket(..)"); - let _nonce_guard = transaction_nonce_guard(&provider).await; - let from_address = provider.provider().default_signer_address(); - let current_nonce = provider - .provider() - .get_transaction_count(from_address) - .pending() - .await?; - let contract = ICiphernodeRegistry::new(contract_address, provider.provider()); - let builder = contract - .submitTicket(e3_id_u256, ticket_number_u256) - .nonce(current_nonce); - let pending = builder.send().await?; - drop(_nonce_guard); - let receipt = pending.get_receipt().await?; - require_successful_receipt("submit ticket", &receipt)?; - Ok(receipt) - } - }) - .await -} - -pub async fn finalize_committee_on_registry( - provider: EthProvider

, - contract_address: Address, - e3_id: E3id, -) -> Result { - let e3_id_u256: U256 = e3_id.try_into()?; - - send_tx_with_retry( - "finalizeCommittee", - &[ - "SubmissionWindowNotClosed", - "CommitteeNotRequested", - "ThresholdNotMet", - ], - || { - let provider = provider.clone(); - async move { - info!("Calling: contract.finalizeCommittee(..)"); - let _nonce_guard = transaction_nonce_guard(&provider).await; - let from_address = provider.provider().default_signer_address(); - let current_nonce = provider - .provider() - .get_transaction_count(from_address) - .pending() - .await?; - let contract = ICiphernodeRegistry::new(contract_address, provider.provider()); - let builder = contract.finalizeCommittee(e3_id_u256).nonce(current_nonce); - let pending = builder.send().await?; - drop(_nonce_guard); - let receipt = pending.get_receipt().await?; - require_successful_receipt("finalize committee", &receipt)?; - Ok(receipt) - } - }, - ) - .await -} - -async fn should_finalize_committee( - provider: EthProvider

, - contract_address: Address, - e3_id: E3id, -) -> Result { - let e3_id_u256: U256 = e3_id.try_into()?; - let contract = ICiphernodeRegistry::new(contract_address, provider.provider()); - if contract.isOpen(e3_id_u256).call().await? { - return Ok(false); - } - - match contract.finalizeCommittee(e3_id_u256).call().await { - Ok(_) => Ok(true), - Err(err) => { - let err = anyhow::Error::from(err); - let decoded = decode_error_from_str(&format!("{err:?}")); - - if decoded.as_deref().is_some_and(|message| { - message.contains("CommitteeAlreadyFinalized") - || message.contains("CommitteeNotRequested") - || message.contains("SubmissionWindowNotClosed") - || message.contains("ThresholdNotMet") - }) { - return Ok(false); - } - - Err(err) - } - } -} - -async fn should_publish_committee( - provider: EthProvider

, - contract_address: Address, - e3_id: E3id, -) -> Result { - let e3_id_u256: U256 = e3_id.try_into()?; - let contract = ICiphernodeRegistry::new(contract_address, provider.provider()); - match contract.committeePublicKey(e3_id_u256).call().await { - Ok(_) => Ok(false), - Err(err) => { - let err = anyhow::Error::from(err); - let decoded = decode_error_from_str(&format!("{err:?}")); - - if decoded - .as_deref() - .is_some_and(|message| message.contains("CommitteeNotPublished")) - { - return Ok(true); - } - - Err(err) - } - } -} - -pub async fn publish_committee_to_registry( - provider: EthProvider

, - contract_address: Address, - e3_id: E3id, - public_key: ArcBytes, - pk_commitment: [u8; 32], - dkg_aggregator_proof: Option<&Proof>, - dkg_attestation_bundle: Option<&[u8]>, -) -> Result { - let e3_id_u256: U256 = e3_id.try_into()?; - let public_key_bytes = Bytes::from(public_key.extract_bytes()); - let pk_commitment_b256 = B256::from(pk_commitment); - - // `proof` is empty when `proofAggregationEnabled = false`; the contract - // trusts `pk_commitment` directly in that case. - let proof: Bytes = match dkg_aggregator_proof { - Some(p) => encode_zk_proof(p)?, - None => Bytes::new(), - }; - let attestation_bundle: Bytes = match dkg_attestation_bundle { - Some(b) => Bytes::copy_from_slice(b), - None => Bytes::new(), - }; - - // RPC may not have synced finalization yet - send_tx_with_retry("publishCommittee", &["CommitteeNotFinalized"], || { - let provider = provider.clone(); - let public_key_bytes = public_key_bytes.clone(); - let proof = proof.clone(); - let attestation_bundle = attestation_bundle.clone(); - async move { - info!("Calling: contract.publishCommittee(..)"); - let _nonce_guard = transaction_nonce_guard(&provider).await; - let from_address = provider.provider().default_signer_address(); - let current_nonce = provider - .provider() - .get_transaction_count(from_address) - .pending() - .await?; - let contract = ICiphernodeRegistry::new(contract_address, provider.provider()); - let builder = contract - .publishCommittee( - e3_id_u256, - public_key_bytes, - pk_commitment_b256, - proof, - attestation_bundle, - ) - .nonce(current_nonce); - let pending = builder.send().await?; - drop(_nonce_guard); - let receipt = pending.get_receipt().await?; - require_successful_receipt("publish committee", &receipt)?; - Ok(receipt) - } - }) - .await -} - -/// Read `CiphernodeRegistry.dkgFoldAttestationVerifier()` (EIP-712 verifying contract for fold attestations). -pub async fn fetch_dkg_fold_attestation_verifier( - provider: &P, - registry_address: Address, -) -> Result> { - let contract = ICiphernodeRegistry::new(registry_address, provider); - let verifier = contract.dkgFoldAttestationVerifier().call().await?; - if verifier == Address::ZERO { - Ok(None) - } else { - Ok(Some(verifier)) - } -} - -/// Read `CiphernodeRegistry.accusationVoteValidity()` — registry-wide off-chain -/// freshness window (seconds) accusers stamp on `AccusationVote.deadline`. -/// Returns the raw `uint256` as `U256`; callers decide how to clamp it to -/// their own arithmetic type. `Ok(None)` is reserved for the case where the -/// registry has been governance-disabled (`accusationVoteValidity = 0`) so -/// the caller can short-circuit without producing votes that will never -/// verify on chain. -pub async fn fetch_accusation_vote_validity( - provider: &P, - registry_address: Address, -) -> Result> { - let contract = ICiphernodeRegistry::new(registry_address, provider); - let validity = contract.accusationVoteValidity().call().await?; - if validity.is_zero() { - Ok(None) - } else { - Ok(Some(validity)) - } -} - -/// Wrapper for a reader and writer -pub struct CiphernodeRegistrySol; - -impl CiphernodeRegistrySol { - pub fn attach(processor: &Recipient) -> Addr { - CiphernodeRegistrySolReader::setup(processor) - } - - pub fn attach_writer

(bus: &BusHandle, provider: EthProvider

, contract_address: Address) - where - P: Provider + WalletProvider + Clone + 'static, - { - CiphernodeRegistrySolWriter::attach(bus, provider, contract_address); - } -} diff --git a/crates/evm/src/actors/evm_chain_gateway.rs b/crates/evm/src/actors/evm_chain_gateway.rs deleted file mode 100644 index 5ce870b06d..0000000000 --- a/crates/evm/src/actors/evm_chain_gateway.rs +++ /dev/null @@ -1,455 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::domain::chain_sync_state::SyncStatus; -use crate::messages::HistoricalSyncComplete; -use crate::messages::InterfoldEvmEvent; -use actix::{Actor, ActorContext, Handler}; -use actix::{Addr, Recipient}; -use anyhow::{bail, Context, Result}; -use e3_events::EType; -use e3_events::{ - BusHandle, ErrorDispatcher, EventSubscriber, EventType, HistoricalEvmEventsReceived, - HistoricalEvmSyncStart, InterfoldEvent, InterfoldEventData, SyncEnded, Unsequenced, -}; -use e3_events::{Event, EventPublisher}; -use e3_utils::MAILBOX_LIMIT; -use tokio::sync::oneshot; -use tracing::warn; - -/// Per-chain bound for events accumulated while the node is synchronizing. -/// -/// Tests inject a smaller value. Production deliberately fails startup instead -/// of dropping an observed chain event if this window is exhausted. -pub const DEFAULT_MAX_BUFFERED_EVM_EVENTS: usize = 100_000; - -pub struct EvmChainGatewayHandle { - addr: Addr, - readiness: oneshot::Receiver>, -} - -impl EvmChainGatewayHandle { - pub fn addr(&self) -> Addr { - self.addr.clone() - } - - pub async fn wait_until_live(self) -> Result<()> { - self.readiness - .await - .context("EVM chain gateway stopped before reporting startup status")? - .map_err(anyhow::Error::msg) - } -} - -/// This component sits between the Evm ingestion for a chain and the Sync actor and the Bus. -/// It coordinates event flow between these components. -pub struct EvmChainGateway { - bus: BusHandle, - status: SyncStatus>, - max_buffered_events: usize, - readiness: Option>>, -} - -impl EvmChainGateway { - pub fn new(bus: &BusHandle) -> Self { - Self::with_options(bus, DEFAULT_MAX_BUFFERED_EVM_EVENTS, None) - } - - fn with_options( - bus: &BusHandle, - max_buffered_events: usize, - readiness: Option>>, - ) -> Self { - Self { - bus: bus.clone(), - status: SyncStatus::default(), - max_buffered_events, - readiness, - } - } - - pub fn setup(bus: &BusHandle) -> Addr { - Self::start_and_subscribe(bus, Self::new(bus)) - } - - pub fn setup_with_readiness(bus: &BusHandle) -> EvmChainGatewayHandle { - Self::setup_with_readiness_and_limit(bus, DEFAULT_MAX_BUFFERED_EVM_EVENTS) - } - - pub fn setup_with_readiness_and_limit( - bus: &BusHandle, - max_buffered_events: usize, - ) -> EvmChainGatewayHandle { - let (tx, readiness) = oneshot::channel(); - let actor = Self::with_options(bus, max_buffered_events, Some(tx)); - let addr = Self::start_and_subscribe(bus, actor); - EvmChainGatewayHandle { addr, readiness } - } - - fn start_and_subscribe(bus: &BusHandle, actor: Self) -> Addr { - let addr = actor.start(); - bus.subscribe_all( - &[EventType::HistoricalEvmSyncStart, EventType::SyncEnded], - addr.clone().recipient(), - ); - addr - } - - fn signal_startup(&mut self, result: std::result::Result<(), String>) { - if let Some(sender) = self.readiness.take() { - let _ = sender.send(result); - } - } - - fn fail_closed(&mut self, error: anyhow::Error, ctx: &mut actix::Context) { - let reason = format!( - "EVM chain gateway failed closed: {error:#}. The gateway stopped and will not process \ - further chain events; inspect the snapshot/deploy block and RPC catch-up range, then \ - restart the node to replay chain history" - ); - self.status.fail(reason.clone()); - self.signal_startup(Err(reason.clone())); - self.bus.err(EType::Evm, anyhow::anyhow!(reason)); - ctx.stop(); - } - - fn handle_sync_start(&mut self, msg: HistoricalEvmSyncStart) -> Result<()> { - let sender = msg - .sender - .context("No sender on HistoricalEvmSyncStart Message")?; - let (mut buffer, pending_sync_complete) = self.status.forward_to_sync_actor(sender)?; - - for evt in buffer.drain(..) { - self.process_evm_event(evt)?; - } - - // HistoricalSyncComplete may have arrived before HistoricalEvmSyncStart - if let Some(event) = pending_sync_complete { - warn!("Processing buffered HistoricalSyncComplete that arrived during Init"); - self.forward_historical_sync_complete(event)?; - } - Ok(()) - } - - fn handle_sync_ended(&mut self, _: SyncEnded) -> Result<()> { - let buffer = self.status.live()?; - for evt in buffer { - self.publish_evm_event(evt)?; - } - self.signal_startup(Ok(())); - Ok(()) - } - - fn publish_evm_event(&mut self, msg: InterfoldEvent) -> Result<()> { - self.bus.naked_dispatch(msg); - Ok(()) - } - - fn handle_evm_event(&mut self, msg: InterfoldEvmEvent) -> Result<()> { - match msg { - InterfoldEvmEvent::HistoricalSyncComplete(e) => { - self.forward_historical_sync_complete(e)?; - Ok(()) - } - InterfoldEvmEvent::Event(event) => { - self.process_evm_event(event.into_interfold_event(&self.bus)?)?; - Ok(()) - } - InterfoldEvmEvent::Log(_) => { - bail!("EvmChainGateway received an unparsed EVM log") - } - InterfoldEvmEvent::Rejected(rejected) => bail!( - "chain {} rejected provider log {}: {}", - rejected.chain_id, - rejected.id, - rejected.reason - ), - InterfoldEvmEvent::Processed(_) => { - bail!("EvmChainGateway received an internal ordering marker") - } - } - } - - fn forward_historical_sync_complete(&mut self, event: HistoricalSyncComplete) -> Result<()> { - // Buffer if we're still in Init - will be replayed when HistoricalEvmSyncStart arrives - if let SyncStatus::Init { - pending_sync_complete, - .. - } = &mut self.status - { - warn!( - chain_id = event.chain_id, - "HistoricalSyncComplete arrived during Init, buffering" - ); - *pending_sync_complete = Some(event); - return Ok(()); - } - - let state = self.status.buffer_until_live()?; - let sender = state - .sender - .context("ForwardToSyncActor state must hold a sender")?; - let event = HistoricalEvmEventsReceived::new(state.buffer, event.chain_id); - sender.try_send(event)?; - Ok(()) - } - - fn process_evm_event(&mut self, msg: InterfoldEvent) -> Result<()> { - if matches!(self.status, SyncStatus::Live) { - return self.publish_evm_event(msg); - } - self.status - .add_buffered_event(msg, self.max_buffered_events) - } -} - -impl Actor for EvmChainGateway { - type Context = actix::Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT) - } - - fn stopped(&mut self, _: &mut Self::Context) { - self.signal_startup(Err( - "EVM chain gateway stopped before reaching Live; inspect preceding EVM errors" - .to_owned(), - )); - } -} - -impl Handler for EvmChainGateway { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let result = (|| { - match msg.into_data() { - InterfoldEventData::HistoricalEvmSyncStart(e) => self.handle_sync_start(e)?, - InterfoldEventData::SyncEnded(e) => self.handle_sync_ended(e)?, - _ => (), - } - Ok(()) - })(); - if let Err(error) = result { - self.fail_closed(error, ctx); - } - } -} - -impl Handler for EvmChainGateway { - type Result = (); - fn handle(&mut self, msg: InterfoldEvmEvent, ctx: &mut Self::Context) -> Self::Result { - if let Err(error) = self.handle_evm_event(msg) { - self.fail_closed(error, ctx); - } - } -} - -#[cfg(test)] -mod tests { - use crate::{EvmEvent, EvmLogRejected}; - - use super::*; - use e3_ciphernode_builder::EventSystem; - - use e3_events::{CorrelationId, EvmEventConfig, EvmEventConfigChain, TakeEvents, TestEvent}; - use tokio::sync::mpsc; - use tracing_subscriber::{fmt, EnvFilter}; - - struct SyncEventCollector { - tx: mpsc::UnboundedSender, - } - - #[actix::test] - async fn rejected_log_fails_gateway_readiness() -> Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("test-rejected-log"); - let gateway = EvmChainGateway::setup_with_readiness(&bus); - - gateway - .addr() - .send(InterfoldEvmEvent::Rejected(EvmLogRejected::new( - CorrelationId::new(), - 1, - "malformed historical log", - ))) - .await?; - - let error = gateway.wait_until_live().await.unwrap_err(); - assert!(error.to_string().contains("malformed historical log")); - Ok(()) - } - - impl Actor for SyncEventCollector { - type Context = actix::Context; - } - - impl Handler for SyncEventCollector { - type Result = (); - fn handle(&mut self, msg: HistoricalEvmEventsReceived, _: &mut Self::Context) { - let _ = self.tx.send(msg); - } - } - - #[actix::test] - async fn test_evm_chain_gateway() -> Result<()> { - let _foo = tracing::subscriber::set_default( - fmt() - .with_env_filter(EnvFilter::new("info")) - .with_test_writer() - .finish(), - ); - - let system = EventSystem::new().with_fresh_bus(); - let bus: BusHandle = system.handle()?.enable("test"); - - let history_collector = bus.history(); - - let (tx, mut rx) = mpsc::unbounded_channel(); - let collector = SyncEventCollector { tx }.start(); - - let gateway = EvmChainGateway::setup_with_readiness(&bus); - let addr = gateway.addr(); - - let chain_id = 1u64; - - // HistoricalEvmSyncStart: Init -> ForwardToSyncActor - let mut evm_config = EvmEventConfig::new(); - evm_config.insert(chain_id, EvmEventConfigChain::new(0)); - bus.publish_without_context(HistoricalEvmSyncStart::new(collector.clone(), evm_config)) - .unwrap(); - - // Send EVM event while forwarding - should reach collector - let evm_event = EvmEvent::new( - CorrelationId::new(), - TestEvent::new("Before Complete", 1).into(), - 100, - 12345, - chain_id, - ); - - // This will actually arrive earlier than HistoricalEvmSyncStart but aught to be buffered - addr.send(InterfoldEvmEvent::Event(evm_event)).await?; - - // HistoricalSyncComplete: ForwardToSyncActor -> BufferUntilLive - addr.send(InterfoldEvmEvent::HistoricalSyncComplete( - HistoricalSyncComplete::new(chain_id, None), - )) - .await?; - - // Normal Synchronizer will take this and wait for other events before flushing events to - // the bus here we simulate it - let received = rx.recv().await.unwrap(); - for event in received.events { - bus.naked_dispatch(event); - } - - // Send EVM event while buffering - should be buffered (not received) - let buffered_event = EvmEvent::new( - CorrelationId::new(), - TestEvent::new("Before SyncEnded", 2).into(), - 101, - 12346, - chain_id, - ); - addr.send(InterfoldEvmEvent::Event(buffered_event)).await?; - - // The Synchronizer will publish the SyncEnded event when it has all the information it needs - // and has published everything to the bus - bus.publish_without_context(SyncEnded::new())?; - gateway.wait_until_live().await?; - - let after_event = EvmEvent::new( - CorrelationId::new(), - TestEvent::new("After SyncEnded", 2).into(), - 101, - 12346, - chain_id, - ); - - addr.send(InterfoldEvmEvent::Event(after_event)).await?; - - let full = history_collector.send(TakeEvents::new(5)).await?; - - let test_events: Vec = full - .events - .iter() - .filter_map(|e| { - if let InterfoldEventData::TestEvent(TestEvent { msg, .. }) = e.get_data() { - Some(msg.to_string()) - } else { - None - } - }) - .collect(); - - assert_eq!( - test_events, - vec!["Before Complete", "Before SyncEnded", "After SyncEnded"] - ); - - let event_types: Vec = full.events.iter().map(|e| e.event_type()).collect(); - - assert_eq!( - event_types, - vec![ - "HistoricalEvmSyncStart", - "TestEvent", - "SyncEnded", - "TestEvent", - "TestEvent" - ] - ); - Ok(()) - } - - #[actix::test] - async fn overflow_emits_actionable_error_stops_and_fails_readiness() -> Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let bus: BusHandle = system.handle()?.enable("test-overflow"); - let errors = bus.errors(); - let gateway = EvmChainGateway::setup_with_readiness_and_limit(&bus, 1); - let addr = gateway.addr(); - - for entropy in [1, 2] { - let event = EvmEvent::new( - CorrelationId::new(), - TestEvent::new("overflow", entropy).into(), - 100, - u128::from(entropy), - 1, - ); - addr.send(InterfoldEvmEvent::Event(event)).await?; - } - - let startup_error = gateway - .wait_until_live() - .await - .expect_err("overflow must fail gateway readiness") - .to_string(); - assert!(startup_error.contains("Init buffer reached its limit of 1 events")); - assert!(startup_error.contains("will not process further chain events")); - assert!(startup_error.contains("restart the node to replay chain history")); - - let received = errors.send(TakeEvents::new(1)).await?; - assert!(!received.timed_out, "overflow error should be observable"); - let InterfoldEventData::InterfoldError(error) = received.events[0].get_data() else { - panic!("expected an InterfoldError event"); - }; - assert!(error - .message - .contains("Init buffer reached its limit of 1 events")); - assert!(error.message.contains("snapshot/deploy block")); - - tokio::time::timeout(std::time::Duration::from_secs(1), async { - while addr.connected() { - tokio::task::yield_now().await; - } - }) - .await - .context("overflowed gateway did not stop")?; - assert!(!addr.connected(), "overflowed gateway must stop"); - Ok(()) - } -} diff --git a/crates/evm/src/actors/evm_read_interface.rs b/crates/evm/src/actors/evm_read_interface.rs deleted file mode 100644 index 95e03ceb93..0000000000 --- a/crates/evm/src/actors/evm_read_interface.rs +++ /dev/null @@ -1,526 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::actors::log_fetcher::{ - backfill_to_head, fetch_logs_chunked, process_live_log, TimestampTracker, -}; -use crate::domain::backoff::Backoff; -use crate::helpers::{EthProvider, ProviderFactory}; -use crate::messages::HistoricalSyncComplete; -use crate::messages::{EvmEventProcessor, InterfoldEvmEvent}; -use actix::prelude::*; -use alloy::eips::BlockNumberOrTag; -use alloy::primitives::B256; -use alloy::providers::Provider; -use alloy::rpc::types::Filter; -use alloy_primitives::Address; -use anyhow::anyhow; -use e3_events::{ - BusHandle, EType, ErrorDispatcher, Event, EventId, InterfoldEvent, InterfoldEventData, -}; -use e3_events::{EventSubscriber, EventType}; -use e3_utils::{retry_with_backoff, RetryError, MAILBOX_LIMIT}; -use futures_util::stream::StreamExt; -use std::collections::{HashMap, HashSet}; -use std::time::Duration; -use tokio::select; -use tokio::sync::oneshot; -use tracing::{error, info, instrument, warn}; - -const MAX_RECONNECT_DELAY_SECS: u64 = 60; -/// Maximum attempts to recreate a provider via the factory before adding an -/// extra outer delay. -const PROVIDER_RECREATE_MAX_ATTEMPTS: u32 = 3; -/// Initial delay (ms) between provider-recreation attempts. -const PROVIDER_RECREATE_INITIAL_DELAY_MS: u64 = 2000; -/// Consecutive failures before we assume the provider is dead and recreate it. -const MAX_RETRIES_BEFORE_RECREATE: u32 = 3; -/// Polling is required even while the subscription is quiet: a log becomes confirmed because later -/// blocks arrive, and those blocks need not contain any matching contract event. -const CONFIRMED_BACKFILL_INTERVAL_SECS: u64 = 5; - -#[derive(Default, serde::Serialize, serde::Deserialize, Clone)] -pub struct EvmReadInterfaceState { - pub ids: HashSet, - pub last_block: Option, -} - -#[derive(Clone, Default)] -pub struct Filters { - historical: Filter, - current: Filter, - start_block: u64, - /// Number of confirmations required before a log is ingested. `0` (default) - /// reads to the chain head exactly as before; a positive value clamps the - /// historical and backfill heads to make ingestion reorg-safe. - confirmations: u64, -} - -impl Filters { - pub fn new(addresses: Vec

, start_block: u64) -> Self { - let historical = Filter::new() - .address(addresses.clone()) - .from_block(start_block); - let current = Filter::new() - .address(addresses) - .from_block(BlockNumberOrTag::Latest); - - Self { - historical, - current, - start_block, - confirmations: 0, - } - } - - /// Builder: require `confirmations` block confirmations before ingestion. - pub fn with_confirmations(mut self, confirmations: u64) -> Self { - self.confirmations = confirmations; - self - } - - /// The configured confirmation depth (0 = read to head). - pub fn confirmations(&self) -> u64 { - self.confirmations - } - - pub fn from_routing_table(table: &HashMap, start_block: u64) -> Self { - let addresses: Vec
= table.keys().cloned().collect(); - Self::new(addresses, start_block) - } -} - -/// Connects to Interfold.sol converting EVM events to InterfoldEvents -pub struct EvmReadInterface

{ - /// The alloy provider - provider: Option>, - /// Optional factory to recreate the provider when the transport dies - provider_factory: Option>, - /// A shutdown receiver to listen to for shutdown signals sent to the loop this is only used - /// internally. You should send the Shutdown signal to the reader directly or via the EventBus - shutdown_rx: Option>, - /// The sender for the shutdown signal this is only used internally - shutdown_tx: Option>, - /// Processor to forward events - next: EvmEventProcessor, - /// Event bus for error propagation only - bus: BusHandle, - /// Filters to configure when to seek from - filters: Filters, -} - -impl EvmReadInterface

{ - pub fn setup( - provider: &EthProvider

, - next: impl Into, - bus: &BusHandle, - filters: Filters, - ) -> Addr { - Self::setup_with_factory(provider, None, next, bus, filters) - } - - pub fn setup_with_factory( - provider: &EthProvider

, - provider_factory: Option>, - next: impl Into, - bus: &BusHandle, - filters: Filters, - ) -> Addr { - let (shutdown_tx, shutdown_rx) = oneshot::channel(); - let reader = Self { - provider: Some(provider.clone()), - provider_factory, - shutdown_rx: Some(shutdown_rx), - shutdown_tx: Some(shutdown_tx), - next: next.into(), - bus: bus.clone(), - filters, - }; - - let addr = reader.start(); - bus.subscribe(EventType::Shutdown, addr.clone().into()); - addr - } -} - -impl Actor for EvmReadInterface

{ - type Context = actix::Context; - - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT); - - let bus = self.bus.clone(); - let next = self.next.clone(); - let filters = self.filters.clone(); - let provider_factory = self.provider_factory.take(); - - let Some(provider) = self.provider.take() else { - error!("Could not start event reader as provider has already been used."); - return; - }; - - let Some(shutdown) = self.shutdown_rx.take() else { - bus.err(EType::Evm, anyhow!("shutdown already called")); - return; - }; - - ctx.spawn( - async move { - stream_from_evm(provider, provider_factory, next, shutdown, &bus, filters).await - } - .into_actor(self), - ); - } -} - -async fn sleep_or_shutdown(duration: Duration, shutdown: &mut oneshot::Receiver<()>) -> bool { - select! { - _ = tokio::time::sleep(duration) => false, - _ = &mut *shutdown => { - info!("Shutdown signal received during backoff"); - true - } - } -} - -async fn recreate_provider( - factory: &ProviderFactory

, - shutdown: &mut oneshot::Receiver<()>, - chain_id: u64, - backoff: &mut Backoff, -) -> Option> { - loop { - if shutdown.try_recv().is_ok() { - return None; - } - - let delay = backoff.next_delay(); - warn!( - chain_id, - delay_secs = delay.as_secs(), - "Waiting before provider recreation attempt" - ); - if sleep_or_shutdown(delay, shutdown).await { - return None; - } - - let factory_clone = factory.clone(); - let result = retry_with_backoff( - || { - let f = factory_clone.clone(); - async move { - let provider = f().await.map_err(|e| { - warn!(chain_id, error = %e, "Factory failed to create provider"); - RetryError::Retry(e) - })?; - - // Health check: verify the new transport is actually alive - provider.provider().get_block_number().await.map_err(|e| { - warn!(chain_id, error = %e, "New provider failed health check"); - RetryError::Retry(anyhow!("Health check failed: {}", e)) - })?; - - Ok(provider) - } - }, - PROVIDER_RECREATE_MAX_ATTEMPTS, - PROVIDER_RECREATE_INITIAL_DELAY_MS, - ) - .await; - - match result { - Ok(new_provider) => { - let new_chain_id = new_provider.chain_id(); - if new_chain_id != chain_id { - error!( - chain_id, - new_chain_id, "Recreated provider is on wrong chain — fatal" - ); - return None; - } - info!(chain_id, "Provider recreated and verified"); - backoff.reset(); - return Some(new_provider); - } - Err(e) => { - error!( - chain_id, - error = %e, - "All provider recreation attempts failed, will retry with longer backoff" - ); - continue; - } - } - } -} - -async fn get_new_provider_or_exit( - factory: &Option>, - shutdown: &mut oneshot::Receiver<()>, - chain_id: u64, - backoff: &mut Backoff, - bus: &BusHandle, -) -> Option> { - let Some(factory) = factory else { - error!( - chain_id, - "Transport died and no provider factory configured" - ); - bus.err( - EType::Evm, - anyhow!("Transport died and no provider factory configured"), - ); - return None; - }; - let result = recreate_provider(factory, shutdown, chain_id, backoff).await; - if result.is_none() && shutdown.try_recv().is_err() { - bus.err( - EType::Evm, - anyhow!("Provider recreation failed for chain {}", chain_id), - ); - } - result -} - -#[instrument(name = "evm_interface", skip_all)] -async fn stream_from_evm( - provider: EthProvider

, - provider_factory: Option>, - next: EvmEventProcessor, - mut shutdown: oneshot::Receiver<()>, - bus: &BusHandle, - filters: Filters, -) { - let chain_id = provider.chain_id(); - let mut timestamp_tracker = TimestampTracker::new(); - let mut backoff = Backoff::new(MAX_RECONNECT_DELAY_SECS); - - // ── Phase 1: Historical sync (must succeed, fatal on failure) ── - - let latest_block = match provider.provider().get_block_number().await { - Ok(bn) => crate::domain::reorg::confirmed_head(bn, filters.confirmations()), - Err(e) => { - error!(chain_id, error = %e, "Failed to get latest block number"); - bus.err(EType::Evm, anyhow!(e)); - return; - } - }; - - let last_id = match fetch_logs_chunked( - provider.provider(), - &filters.historical, - filters.start_block, - latest_block, - chain_id, - &next, - &mut timestamp_tracker, - ) - .await - { - Ok(id) => { - info!(chain_id, "Historical sync succeeded"); - id - } - Err(e) => { - error!(chain_id, error = %e, "Failed to fetch historical events — node cannot operate without full state, exiting"); - bus.err(EType::Evm, anyhow!(e)); - return; - } - }; - - next.do_send(InterfoldEvmEvent::HistoricalSyncComplete( - HistoricalSyncComplete::new(chain_id, last_id), - )); - - // ── Phase 2: Live event loop with provider lifecycle management ── - // - // Single flat loop: backfill → subscribe → consume stream → repeat. - // On transport death, immediately recreate the provider. - // On transient errors, retry with exponential backoff. - - let mut last_block = latest_block; - let mut current_provider = provider; - let mut consecutive_failures: u32 = 0; - - loop { - // Step 1: Backfill any blocks missed since last_block - match backfill_to_head( - current_provider.provider(), - &filters.current, - chain_id, - &next, - &mut timestamp_tracker, - &mut last_block, - filters.confirmations(), - ) - .await - { - Ok(_) => { - backoff.reset(); - consecutive_failures = 0; - } - Err(e) => { - consecutive_failures += 1; - warn!(chain_id, error = %e, consecutive_failures, "Backfill failed"); - if consecutive_failures >= MAX_RETRIES_BEFORE_RECREATE { - let Some(p) = get_new_provider_or_exit( - &provider_factory, - &mut shutdown, - chain_id, - &mut backoff, - bus, - ) - .await - else { - return; - }; - current_provider = p; - consecutive_failures = 0; - continue; - } - if sleep_or_shutdown(backoff.next_delay(), &mut shutdown).await { - return; - } - continue; - } - } - - // Step 2: Subscribe to live events - let sub_result = current_provider - .provider() - .subscribe_logs(&filters.current) - .await - .map_err(|e| anyhow!("{}", e)); - - match sub_result { - Ok(subscription) => { - backoff.reset(); - consecutive_failures = 0; - let sub_id: B256 = *subscription.local_id(); - let mut stream = subscription.into_stream(); - let mut confirmation_poll = - tokio::time::interval(Duration::from_secs(CONFIRMED_BACKFILL_INTERVAL_SECS)); - confirmation_poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); - // Tokio intervals tick immediately once. The outer loop already backfilled, so - // consume that tick and wait for the configured period before querying again. - confirmation_poll.tick().await; - info!(chain_id, "Live event subscription active"); - - loop { - select! { - maybe_log = stream.next() => { - match maybe_log { - Some(log) => { - if let Err(error) = process_live_log( - current_provider.provider(), log, chain_id, &next, - &mut timestamp_tracker, &mut last_block, - filters.confirmations(), - ).await { - consecutive_failures += 1; - warn!( - chain_id, - error = %error, - consecutive_failures, - "Live log rejected; reconnecting for canonical backfill" - ); - break; - } - } - None => { - // Stream ended (server-side close, idle timeout, etc.) - consecutive_failures += 1; - warn!(chain_id, consecutive_failures, "Live event stream ended, will reconnect"); - break; - } - } - } - _ = confirmation_poll.tick(), if filters.confirmations() > 0 => { - if let Err(error) = backfill_to_head( - current_provider.provider(), - &filters.current, - chain_id, - &next, - &mut timestamp_tracker, - &mut last_block, - filters.confirmations(), - ).await { - consecutive_failures += 1; - warn!( - chain_id, - error = %error, - consecutive_failures, - "Confirmed live-log backfill failed; reconnecting" - ); - break; - } - } - _ = &mut shutdown => { - info!("Shutdown signal received, stopping EVM stream"); - let _ = current_provider.provider().unsubscribe(sub_id).await; - return; - } - } - } - - if consecutive_failures >= MAX_RETRIES_BEFORE_RECREATE { - let Some(p) = get_new_provider_or_exit( - &provider_factory, - &mut shutdown, - chain_id, - &mut backoff, - bus, - ) - .await - else { - return; - }; - current_provider = p; - consecutive_failures = 0; - } else if consecutive_failures > 0 - && sleep_or_shutdown(backoff.next_delay(), &mut shutdown).await - { - return; - } - } - Err(e) => { - consecutive_failures += 1; - error!(chain_id, error = %e, consecutive_failures, "Failed to subscribe to live events"); - if consecutive_failures >= MAX_RETRIES_BEFORE_RECREATE { - let Some(p) = get_new_provider_or_exit( - &provider_factory, - &mut shutdown, - chain_id, - &mut backoff, - bus, - ) - .await - else { - return; - }; - current_provider = p; - consecutive_failures = 0; - } else { - bus.err(EType::Evm, anyhow!("{}", e)); - if sleep_or_shutdown(backoff.next_delay(), &mut shutdown).await { - return; - } - } - } - } - } -} - -impl Handler for EvmReadInterface

{ - type Result = (); - - fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { - if let InterfoldEventData::Shutdown(_) = msg.into_data() { - if let Some(shutdown) = self.shutdown_tx.take() { - let _ = shutdown.send(()); - } - } - } -} diff --git a/crates/evm/src/actors/log_fetcher.rs b/crates/evm/src/actors/log_fetcher.rs deleted file mode 100644 index 192eb031e4..0000000000 --- a/crates/evm/src/actors/log_fetcher.rs +++ /dev/null @@ -1,672 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::messages::{EvmEventProcessor, EvmLog, InterfoldEvmEvent}; -use alloy::providers::Provider; -use alloy::rpc::types::{Filter, Log}; -use anyhow::{anyhow, Context as _}; -use async_trait::async_trait; -use e3_events::CorrelationId; -use tracing::{debug, info, warn}; - -const GET_LOGS_CHUNK_SIZE: u64 = 10_000; -const GET_LOGS_MAX_RETRIES: u32 = 3; - -/// Trait abstracting provider methods needed for log fetching. -/// Enables unit testing without a real EVM provider. -#[async_trait] -pub(crate) trait LogProvider: Send + Sync { - async fn fetch_logs(&self, filter: &Filter) -> Result, anyhow::Error>; - async fn fetch_block_number(&self) -> Result; - async fn fetch_block_timestamp(&self, block_number: u64) -> Result; -} - -#[async_trait] -impl LogProvider for P { - async fn fetch_logs(&self, filter: &Filter) -> Result, anyhow::Error> { - self.get_logs(filter).await.map_err(|e| anyhow!("{}", e)) - } - async fn fetch_block_number(&self) -> Result { - self.get_block_number().await.map_err(|e| anyhow!("{}", e)) - } - async fn fetch_block_timestamp(&self, block_number: u64) -> Result { - self.get_block_by_number(block_number.into()) - .await - .map_err(|error| anyhow!("failed to fetch block {block_number}: {error}"))? - .map(|block| block.header.timestamp) - .ok_or_else(|| anyhow!("provider returned no block for height {block_number}")) - } -} - -pub(crate) async fn process_log( - provider: &L, - log: Log, - chain_id: u64, - next: &EvmEventProcessor, - timestamp_tracker: &mut TimestampTracker, -) -> Result { - let timestamp = timestamp_tracker.get(provider, log.block_number).await?; - let evt = InterfoldEvmEvent::Log(EvmLog::new(log, chain_id, timestamp)); - let id = evt.get_id(); - debug!("Sending event({})", id); - next.do_send(evt); - Ok(id) -} - -/// Handle a log delivered by the subscription stream. -/// -/// With a positive confirmation depth the subscription is only a wake-up signal. Publishing the -/// raw notification here would make the historical confirmation gate ineffective, so the periodic -/// canonical backfill owns delivery instead. With zero confirmations this preserves the existing -/// low-latency behavior. -pub(crate) async fn process_live_log( - provider: &L, - log: Log, - chain_id: u64, - next: &EvmEventProcessor, - timestamp_tracker: &mut TimestampTracker, - last_block: &mut u64, - confirmations: u64, -) -> Result, anyhow::Error> { - if confirmations > 0 { - debug!( - chain_id, - block_number = log.block_number, - confirmations, - "Deferring live log to confirmed canonical backfill" - ); - return Ok(None); - } - - let block_number = log.block_number; - let id = process_log(provider, log, chain_id, next, timestamp_tracker).await?; - if let Some(block_number) = block_number { - *last_block = (*last_block).max(block_number); - } - Ok(Some(id)) -} - -/// Fetch logs in chunks from `from_block` to `to_block` with retry logic per chunk. -/// Returns the CorrelationId of the last processed event, if any. -pub(crate) async fn fetch_logs_chunked( - provider: &L, - filter: &Filter, - from_block: u64, - to_block: u64, - chain_id: u64, - next: &EvmEventProcessor, - timestamp_tracker: &mut TimestampTracker, -) -> Result, anyhow::Error> { - if to_block < from_block { - return Ok(None); - } - - let total_blocks = to_block - from_block + 1; - let total_chunks = total_blocks.div_ceil(GET_LOGS_CHUNK_SIZE); - - info!( - chain_id, - from_block, to_block, total_chunks, "Fetching logs in chunks" - ); - - let mut cursor = from_block; - let mut last_id: Option = None; - let mut chunk_idx = 0u64; - - while cursor <= to_block { - let chunk_end = (cursor + GET_LOGS_CHUNK_SIZE - 1).min(to_block); - chunk_idx += 1; - - let chunk_filter = filter.clone().from_block(cursor).to_block(chunk_end); - - let mut success = false; - for attempt in 1..=GET_LOGS_MAX_RETRIES { - match provider.fetch_logs(&chunk_filter).await { - Ok(logs) => { - info!( - chain_id, - chunk = chunk_idx, - total_chunks, - from = cursor, - to = chunk_end, - events = logs.len(), - "Fetched log chunk" - ); - for log in logs { - last_id = Some( - process_log(provider, log, chain_id, next, timestamp_tracker).await?, - ); - } - success = true; - break; - } - Err(e) => { - warn!( - chain_id, chunk = chunk_idx, - from = cursor, to = chunk_end, - attempt, max_retries = GET_LOGS_MAX_RETRIES, - error = %e, "Failed to fetch log chunk, retrying" - ); - if attempt < GET_LOGS_MAX_RETRIES { - tokio::time::sleep(std::time::Duration::from_secs(2u64.pow(attempt))).await; - } - } - } - } - - if !success { - return Err(anyhow!( - "Failed to fetch logs for chain {} blocks {}..={} after {} retries", - chain_id, - cursor, - chunk_end, - GET_LOGS_MAX_RETRIES - )); - } - - cursor = chunk_end + 1; - } - - info!(chain_id, chunks_fetched = chunk_idx, "Log fetch complete"); - Ok(last_id) -} - -/// Fetch any blocks between `last_block` and the chain head to fill gaps. -/// Handles blocks missed during reconnection or due to Geth's eth_subscribe -/// silently ignoring the fromBlock parameter. -pub(crate) async fn backfill_to_head( - provider: &L, - filter: &Filter, - chain_id: u64, - next: &EvmEventProcessor, - timestamp_tracker: &mut TimestampTracker, - last_block: &mut u64, - confirmations: u64, -) -> Result<(), anyhow::Error> { - let raw_head = provider - .fetch_block_number() - .await - .map_err(|e| anyhow!("Failed to get block number for gap backfill: {}", e))?; - // Clamp to the confirmed head so we never ingest logs that a reorg of depth - // `confirmations` could still orphan. `confirmations == 0` is a no-op. - let current_head = crate::domain::reorg::confirmed_head(raw_head, confirmations); - - let gap_start = *last_block + 1; - if gap_start > current_head { - return Ok(()); - } - - info!( - chain_id, - from = gap_start, - to = current_head, - blocks = current_head - gap_start + 1, - "Backfilling missed blocks" - ); - - let mut cursor = gap_start; - while cursor <= current_head { - let chunk_end = (cursor + GET_LOGS_CHUNK_SIZE - 1).min(current_head); - - fetch_logs_chunked( - provider, - filter, - cursor, - chunk_end, - chain_id, - next, - timestamp_tracker, - ) - .await?; - - *last_block = chunk_end; - cursor = chunk_end + 1; - } - - Ok(()) -} - -/// Cache utility to keep track of timestamps -pub(crate) struct TimestampTracker { - current: Option<(u64, u64)>, // (block_number, timestamp) -} - -impl TimestampTracker { - pub fn new() -> Self { - Self { current: None } - } - - pub async fn get( - &mut self, - provider: &L, - block_number: Option, - ) -> Result { - let bn = block_number.context("provider log is missing its block number")?; - - if let Some((cached_bn, ts)) = self.current { - if bn == cached_bn { - return Ok(ts); - } - } - - let ts = provider.fetch_block_timestamp(bn).await?; - - self.current = Some((bn, ts)); - Ok(ts) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use actix::prelude::*; - use std::collections::VecDeque; - use std::sync::{Arc, Mutex}; - use tokio::sync::mpsc; - - #[derive(Clone)] - struct MockLogProvider { - inner: Arc>, - } - - struct MockState { - block_number: u64, - log_responses: VecDeque, String>>, - timestamp_responses: VecDeque>, - timestamp_calls: u32, - get_logs_calls: u32, - } - - impl MockLogProvider { - fn new(block_number: u64) -> Self { - Self { - inner: Arc::new(Mutex::new(MockState { - block_number, - log_responses: VecDeque::new(), - timestamp_responses: VecDeque::new(), - timestamp_calls: 0, - get_logs_calls: 0, - })), - } - } - - fn push_logs(&self, logs: Vec) { - self.inner.lock().unwrap().log_responses.push_back(Ok(logs)); - } - - fn push_error(&self, msg: &str) { - self.inner - .lock() - .unwrap() - .log_responses - .push_back(Err(msg.to_string())); - } - - fn get_logs_call_count(&self) -> u32 { - self.inner.lock().unwrap().get_logs_calls - } - - fn push_timestamp(&self, timestamp: u64) { - self.inner - .lock() - .unwrap() - .timestamp_responses - .push_back(Ok(timestamp)); - } - - fn push_timestamp_error(&self, message: &str) { - self.inner - .lock() - .unwrap() - .timestamp_responses - .push_back(Err(message.to_string())); - } - - fn timestamp_call_count(&self) -> u32 { - self.inner.lock().unwrap().timestamp_calls - } - - fn set_block_number(&self, block_number: u64) { - self.inner.lock().unwrap().block_number = block_number; - } - } - - #[async_trait] - impl LogProvider for MockLogProvider { - async fn fetch_logs(&self, _filter: &Filter) -> Result, anyhow::Error> { - let mut state = self.inner.lock().unwrap(); - state.get_logs_calls += 1; - match state.log_responses.pop_front() { - Some(Ok(logs)) => Ok(logs), - Some(Err(msg)) => Err(anyhow!("{}", msg)), - None => Ok(vec![]), - } - } - - async fn fetch_block_number(&self) -> Result { - Ok(self.inner.lock().unwrap().block_number) - } - - async fn fetch_block_timestamp(&self, _block_number: u64) -> Result { - let mut state = self.inner.lock().unwrap(); - state.timestamp_calls += 1; - match state.timestamp_responses.pop_front() { - Some(Ok(timestamp)) => Ok(timestamp), - Some(Err(message)) => Err(anyhow!(message)), - None => Ok(0), - } - } - } - - struct TestCollector { - tx: mpsc::UnboundedSender, - } - - impl Actor for TestCollector { - type Context = Context; - } - - impl Handler for TestCollector { - type Result = (); - fn handle(&mut self, msg: InterfoldEvmEvent, _: &mut Self::Context) { - let _ = self.tx.send(msg); - } - } - - fn make_test_log(block_number: u64) -> Log { - Log { - block_number: Some(block_number), - ..Default::default() - } - } - - fn setup_collector() -> ( - EvmEventProcessor, - mpsc::UnboundedReceiver, - ) { - let (tx, rx) = mpsc::unbounded_channel(); - let addr = TestCollector { tx }.start(); - (addr.recipient(), rx) - } - - #[actix::test] - async fn test_fetch_logs_empty_range() { - let mock = MockLogProvider::new(100); - let (next, _rx) = setup_collector(); - let mut ts = TimestampTracker::new(); - let filter = Filter::new(); - - let result = fetch_logs_chunked(&mock, &filter, 200, 100, 1, &next, &mut ts).await; - - assert!(result.is_ok()); - assert!(result.unwrap().is_none()); - assert_eq!(mock.get_logs_call_count(), 0); - } - - #[actix::test] - async fn timestamp_failure_is_not_cached_as_zero() { - let mock = MockLogProvider::new(100); - mock.push_timestamp_error("RPC unavailable"); - mock.push_timestamp(1234); - let mut tracker = TimestampTracker::new(); - - let error = tracker.get(&mock, Some(100)).await.unwrap_err(); - assert!(error.to_string().contains("RPC unavailable")); - assert_eq!(tracker.get(&mock, Some(100)).await.unwrap(), 1234); - assert_eq!(tracker.get(&mock, Some(100)).await.unwrap(), 1234); - assert_eq!(mock.timestamp_call_count(), 2); - } - - #[actix::test] - async fn missing_log_block_number_is_rejected_without_rpc_fallback() { - let mock = MockLogProvider::new(100); - let mut tracker = TimestampTracker::new(); - - let error = tracker.get(&mock, None).await.unwrap_err(); - assert!(error.to_string().contains("missing its block number")); - assert_eq!(mock.timestamp_call_count(), 0); - } - - #[actix::test] - async fn timestamp_rpc_failure_prevents_log_dispatch() { - let mock = MockLogProvider::new(100); - mock.push_logs(vec![make_test_log(100)]); - mock.push_timestamp_error("timestamp RPC unavailable"); - let (next, mut receiver) = setup_collector(); - let mut tracker = TimestampTracker::new(); - - let error = fetch_logs_chunked(&mock, &Filter::new(), 100, 100, 1, &next, &mut tracker) - .await - .unwrap_err(); - - assert!(error.to_string().contains("timestamp RPC unavailable")); - tokio::task::yield_now().await; - assert!(receiver.try_recv().is_err()); - } - - #[actix::test] - async fn test_fetch_logs_single_chunk() { - let mock = MockLogProvider::new(5000); - mock.push_logs(vec![ - make_test_log(100), - make_test_log(200), - make_test_log(300), - ]); - let (next, mut rx) = setup_collector(); - let mut ts = TimestampTracker::new(); - let filter = Filter::new(); - - let result = fetch_logs_chunked(&mock, &filter, 0, 5000, 1, &next, &mut ts).await; - - assert!(result.is_ok()); - assert!(result.unwrap().is_some()); - assert_eq!(mock.get_logs_call_count(), 1); - - // Allow actix message delivery - tokio::task::yield_now().await; - let mut count = 0; - while rx.try_recv().is_ok() { - count += 1; - } - assert_eq!(count, 3); - } - - #[actix::test] - async fn test_fetch_logs_multiple_chunks() { - // 25k blocks → 3 chunks: [0..9999], [10000..19999], [20000..24999] - let mock = MockLogProvider::new(25000); - mock.push_logs(vec![make_test_log(5000)]); - mock.push_logs(vec![make_test_log(15000)]); - mock.push_logs(vec![make_test_log(22000)]); - let (next, _rx) = setup_collector(); - let mut ts = TimestampTracker::new(); - let filter = Filter::new(); - - let result = fetch_logs_chunked(&mock, &filter, 0, 24999, 1, &next, &mut ts).await; - - assert!(result.is_ok()); - assert!(result.unwrap().is_some()); - assert_eq!(mock.get_logs_call_count(), 3); - } - - #[actix::test] - async fn test_fetch_logs_retry_then_success() { - tokio::time::pause(); // Skip retry delays - - let mock = MockLogProvider::new(5000); - mock.push_error("temporary RPC error"); - mock.push_logs(vec![make_test_log(100)]); - let (next, _rx) = setup_collector(); - let mut ts = TimestampTracker::new(); - let filter = Filter::new(); - - let result = fetch_logs_chunked(&mock, &filter, 0, 5000, 1, &next, &mut ts).await; - - assert!(result.is_ok()); - assert!(result.unwrap().is_some()); - assert_eq!(mock.get_logs_call_count(), 2); - } - - #[actix::test] - async fn test_fetch_logs_all_retries_exhausted() { - tokio::time::pause(); - - let mock = MockLogProvider::new(5000); - for _ in 0..GET_LOGS_MAX_RETRIES { - mock.push_error("persistent RPC error"); - } - let (next, _rx) = setup_collector(); - let mut ts = TimestampTracker::new(); - let filter = Filter::new(); - - let result = fetch_logs_chunked(&mock, &filter, 0, 5000, 1, &next, &mut ts).await; - - let err = result.expect_err("expected error after all retries exhausted"); - assert!( - err.to_string().contains("Failed to fetch logs"), - "unexpected error: {err}" - ); - assert_eq!(mock.get_logs_call_count(), GET_LOGS_MAX_RETRIES); - } - - #[actix::test] - async fn test_backfill_no_gap() { - let mock = MockLogProvider::new(100); - let (next, _rx) = setup_collector(); - let mut ts = TimestampTracker::new(); - let filter = Filter::new(); - let mut last_block = 100u64; - - let result = backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 0).await; - - assert!(result.is_ok()); - assert_eq!(last_block, 100); - assert_eq!(mock.get_logs_call_count(), 0); - } - - #[actix::test] - async fn test_backfill_with_gap() { - let mock = MockLogProvider::new(200); - mock.push_logs(vec![make_test_log(150), make_test_log(180)]); - let (next, mut rx) = setup_collector(); - let mut ts = TimestampTracker::new(); - let filter = Filter::new(); - let mut last_block = 100u64; - - let result = backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 0).await; - - assert!(result.is_ok()); - assert_eq!(last_block, 200); - assert_eq!(mock.get_logs_call_count(), 1); - - tokio::task::yield_now().await; - let mut count = 0; - while rx.try_recv().is_ok() { - count += 1; - } - assert_eq!(count, 2); - } - - #[actix::test] - async fn test_backfill_partial_failure_preserves_progress() { - tokio::time::pause(); - - // Head at 25000, last_block at 100 - // Gap: blocks 101..=25000 → 3 chunks: - // chunk 1: [101, 10100] - // chunk 2: [10101, 20100] - // chunk 3: [20101, 25000] - let mock = MockLogProvider::new(25000); - // Chunk 1 succeeds - mock.push_logs(vec![make_test_log(500)]); - // Chunk 2 succeeds - mock.push_logs(vec![make_test_log(15000)]); - // Chunk 3: all retries fail - for _ in 0..GET_LOGS_MAX_RETRIES { - mock.push_error("RPC error"); - } - - let (next, _rx) = setup_collector(); - let mut ts = TimestampTracker::new(); - let filter = Filter::new(); - let mut last_block = 100u64; - - let result = backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 0).await; - - // Should fail because chunk 3 exhausted retries - assert!(result.is_err()); - // But last_block must have advanced past the two successful chunks - assert_eq!(last_block, 20100); - - // On retry: gap_start = 20101, head still 25000 → single chunk succeeds - mock.push_logs(vec![make_test_log(22000)]); - - let result = backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 0).await; - assert!(result.is_ok()); - assert_eq!(last_block, 25000); - } - - #[actix::test] - async fn test_backfill_clamps_to_confirmed_head() { - // Head at 200, but require 12 confirmations => only ingest up to 188. - let mock = MockLogProvider::new(200); - mock.push_logs(vec![make_test_log(150)]); - let (next, _rx) = setup_collector(); - let mut ts = TimestampTracker::new(); - let filter = Filter::new(); - let mut last_block = 100u64; - - let result = backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 12).await; - - assert!(result.is_ok()); - // Advanced only to the confirmed head, not the raw head of 200. - assert_eq!(last_block, 188); - } - - #[actix::test] - async fn live_log_waits_for_confirmed_canonical_backfill() -> anyhow::Result<()> { - let mock = MockLogProvider::new(200); - let (next, mut rx) = setup_collector(); - let mut ts = TimestampTracker::new(); - let filter = Filter::new(); - let mut last_block = 188; - - let delivered = process_live_log( - &mock, - make_test_log(200), - 1, - &next, - &mut ts, - &mut last_block, - 12, - ) - .await?; - assert!(delivered.is_none()); - assert_eq!(last_block, 188); - tokio::task::yield_now().await; - assert!( - rx.try_recv().is_err(), - "unconfirmed log must not be emitted" - ); - - mock.set_block_number(211); - mock.push_logs(Vec::new()); - backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 12).await?; - assert_eq!(last_block, 199); - tokio::task::yield_now().await; - assert!(rx.try_recv().is_err(), "eleven blocks is not enough"); - - mock.set_block_number(212); - mock.push_logs(vec![make_test_log(200)]); - backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 12).await?; - assert_eq!(last_block, 200); - let emitted = tokio::time::timeout(std::time::Duration::from_secs(1), rx.recv()) - .await? - .expect("confirmed log should be emitted"); - assert!(matches!(emitted, InterfoldEvmEvent::Log(_))); - Ok(()) - } -} diff --git a/crates/evm/src/adapters.rs b/crates/evm/src/adapters.rs new file mode 100644 index 0000000000..1f925358a7 --- /dev/null +++ b/crates/evm/src/adapters.rs @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Compatibility view of concrete EVM provider adapters stored by capability. + +#[path = "log_fetching/adapter.rs"] +pub(crate) mod log_fetcher; diff --git a/crates/evm/src/actors/bonding_registry_sol.rs b/crates/evm/src/bonding_registry/actor.rs similarity index 100% rename from crates/evm/src/actors/bonding_registry_sol.rs rename to crates/evm/src/bonding_registry/actor.rs diff --git a/crates/evm/src/domain/bonding_registry_events.rs b/crates/evm/src/bonding_registry/events.rs similarity index 100% rename from crates/evm/src/domain/bonding_registry_events.rs rename to crates/evm/src/bonding_registry/events.rs diff --git a/crates/evm/src/chain_gateway/actor.rs b/crates/evm/src/chain_gateway/actor.rs new file mode 100644 index 0000000000..d4a663c2a4 --- /dev/null +++ b/crates/evm/src/chain_gateway/actor.rs @@ -0,0 +1,214 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::domain::chain_sync_state::SyncStatus; +use crate::messages::HistoricalSyncComplete; +use crate::messages::InterfoldEvmEvent; +use actix::{Actor, ActorContext, Handler}; +use actix::{Addr, Recipient}; +use anyhow::{bail, Context, Result}; +use e3_events::EType; +use e3_events::{ + BusHandle, ErrorDispatcher, EventSubscriber, EventType, HistoricalEvmEventsReceived, + HistoricalEvmSyncStart, InterfoldEvent, InterfoldEventData, SyncEnded, Unsequenced, +}; +use e3_events::{Event, EventPublisher}; +use e3_utils::MAILBOX_LIMIT; +use tokio::sync::oneshot; +use tracing::warn; + +/// Per-chain bound for events accumulated while the node is synchronizing. +/// +/// Tests inject a smaller value. Production deliberately fails startup instead +/// of dropping an observed chain event if this window is exhausted. +pub const DEFAULT_MAX_BUFFERED_EVM_EVENTS: usize = 100_000; + +pub struct EvmChainGatewayHandle { + addr: Addr, + readiness: oneshot::Receiver>, +} + +impl EvmChainGatewayHandle { + pub fn addr(&self) -> Addr { + self.addr.clone() + } + + pub async fn wait_until_live(self) -> Result<()> { + self.readiness + .await + .context("EVM chain gateway stopped before reporting startup status")? + .map_err(anyhow::Error::msg) + } +} + +/// This component sits between the Evm ingestion for a chain and the Sync actor and the Bus. +/// It coordinates event flow between these components. +pub struct EvmChainGateway { + bus: BusHandle, + status: SyncStatus>, + max_buffered_events: usize, + readiness: Option>>, +} + +impl EvmChainGateway { + pub fn new(bus: &BusHandle) -> Self { + Self::with_options(bus, DEFAULT_MAX_BUFFERED_EVM_EVENTS, None) + } + + fn with_options( + bus: &BusHandle, + max_buffered_events: usize, + readiness: Option>>, + ) -> Self { + Self { + bus: bus.clone(), + status: SyncStatus::default(), + max_buffered_events, + readiness, + } + } + + pub fn setup(bus: &BusHandle) -> Addr { + Self::start_and_subscribe(bus, Self::new(bus)) + } + + pub fn setup_with_readiness(bus: &BusHandle) -> EvmChainGatewayHandle { + Self::setup_with_readiness_and_limit(bus, DEFAULT_MAX_BUFFERED_EVM_EVENTS) + } + + pub fn setup_with_readiness_and_limit( + bus: &BusHandle, + max_buffered_events: usize, + ) -> EvmChainGatewayHandle { + let (tx, readiness) = oneshot::channel(); + let actor = Self::with_options(bus, max_buffered_events, Some(tx)); + let addr = Self::start_and_subscribe(bus, actor); + EvmChainGatewayHandle { addr, readiness } + } + + fn start_and_subscribe(bus: &BusHandle, actor: Self) -> Addr { + let addr = actor.start(); + bus.subscribe_all( + &[EventType::HistoricalEvmSyncStart, EventType::SyncEnded], + addr.clone().recipient(), + ); + addr + } + + fn signal_startup(&mut self, result: std::result::Result<(), String>) { + if let Some(sender) = self.readiness.take() { + let _ = sender.send(result); + } + } + + fn fail_closed(&mut self, error: anyhow::Error, ctx: &mut actix::Context) { + let reason = format!( + "EVM chain gateway failed closed: {error:#}. The gateway stopped and will not process \ + further chain events; inspect the snapshot/deploy block and RPC catch-up range, then \ + restart the node to replay chain history" + ); + self.status.fail(reason.clone()); + self.signal_startup(Err(reason.clone())); + self.bus.err(EType::Evm, anyhow::anyhow!(reason)); + ctx.stop(); + } + + fn handle_sync_start(&mut self, msg: HistoricalEvmSyncStart) -> Result<()> { + let sender = msg + .sender + .context("No sender on HistoricalEvmSyncStart Message")?; + let (mut buffer, pending_sync_complete) = self.status.forward_to_sync_actor(sender)?; + + for evt in buffer.drain(..) { + self.process_evm_event(evt)?; + } + + // HistoricalSyncComplete may have arrived before HistoricalEvmSyncStart + if let Some(event) = pending_sync_complete { + warn!("Processing buffered HistoricalSyncComplete that arrived during Init"); + self.forward_historical_sync_complete(event)?; + } + Ok(()) + } + + fn handle_sync_ended(&mut self, _: SyncEnded) -> Result<()> { + let buffer = self.status.live()?; + for evt in buffer { + self.publish_evm_event(evt)?; + } + self.signal_startup(Ok(())); + Ok(()) + } + + fn publish_evm_event(&mut self, msg: InterfoldEvent) -> Result<()> { + self.bus.naked_dispatch(msg); + Ok(()) + } + + fn handle_evm_event(&mut self, msg: InterfoldEvmEvent) -> Result<()> { + match msg { + InterfoldEvmEvent::HistoricalSyncComplete(e) => { + self.forward_historical_sync_complete(e)?; + Ok(()) + } + InterfoldEvmEvent::Event(event) => { + self.process_evm_event(event.into_interfold_event(&self.bus)?)?; + Ok(()) + } + InterfoldEvmEvent::Log(_) => { + bail!("EvmChainGateway received an unparsed EVM log") + } + InterfoldEvmEvent::Rejected(rejected) => bail!( + "chain {} rejected provider log {}: {}", + rejected.chain_id, + rejected.id, + rejected.reason + ), + InterfoldEvmEvent::Processed(_) => { + bail!("EvmChainGateway received an internal ordering marker") + } + } + } + + fn forward_historical_sync_complete(&mut self, event: HistoricalSyncComplete) -> Result<()> { + // Buffer if we're still in Init - will be replayed when HistoricalEvmSyncStart arrives + if let SyncStatus::Init { + pending_sync_complete, + .. + } = &mut self.status + { + warn!( + chain_id = event.chain_id, + "HistoricalSyncComplete arrived during Init, buffering" + ); + *pending_sync_complete = Some(event); + return Ok(()); + } + + let state = self.status.buffer_until_live()?; + let sender = state + .sender + .context("ForwardToSyncActor state must hold a sender")?; + let event = HistoricalEvmEventsReceived::new(state.buffer, event.chain_id); + sender.try_send(event)?; + Ok(()) + } + + fn process_evm_event(&mut self, msg: InterfoldEvent) -> Result<()> { + if matches!(self.status, SyncStatus::Live) { + return self.publish_evm_event(msg); + } + self.status + .add_buffered_event(msg, self.max_buffered_events) + } +} + +#[path = "handlers.rs"] +mod handlers; + +#[cfg(test)] +#[path = "tests.rs"] +mod tests; diff --git a/crates/evm/src/chain_gateway/handlers.rs b/crates/evm/src/chain_gateway/handlers.rs new file mode 100644 index 0000000000..78e29d14b5 --- /dev/null +++ b/crates/evm/src/chain_gateway/handlers.rs @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Mailbox entry points and lifecycle hooks. + +use super::*; + +impl Actor for EvmChainGateway { + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } + + fn stopped(&mut self, _: &mut Self::Context) { + self.signal_startup(Err( + "EVM chain gateway stopped before reaching Live; inspect preceding EVM errors" + .to_owned(), + )); + } +} + +impl Handler for EvmChainGateway { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let result = (|| { + match msg.into_data() { + InterfoldEventData::HistoricalEvmSyncStart(e) => self.handle_sync_start(e)?, + InterfoldEventData::SyncEnded(e) => self.handle_sync_ended(e)?, + _ => (), + } + Ok(()) + })(); + if let Err(error) = result { + self.fail_closed(error, ctx); + } + } +} + +impl Handler for EvmChainGateway { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvmEvent, ctx: &mut Self::Context) -> Self::Result { + if let Err(error) = self.handle_evm_event(msg) { + self.fail_closed(error, ctx); + } + } +} diff --git a/crates/evm/src/domain/chain_sync_state.rs b/crates/evm/src/chain_gateway/state.rs similarity index 100% rename from crates/evm/src/domain/chain_sync_state.rs rename to crates/evm/src/chain_gateway/state.rs diff --git a/crates/evm/src/chain_gateway/tests.rs b/crates/evm/src/chain_gateway/tests.rs new file mode 100644 index 0000000000..c0545068ba --- /dev/null +++ b/crates/evm/src/chain_gateway/tests.rs @@ -0,0 +1,210 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::{EvmEvent, EvmLogRejected}; + +use super::*; +use e3_ciphernode_builder::EventSystem; + +use e3_events::{CorrelationId, EvmEventConfig, EvmEventConfigChain, TakeEvents, TestEvent}; +use tokio::sync::mpsc; +use tracing_subscriber::{fmt, EnvFilter}; + +struct SyncEventCollector { + tx: mpsc::UnboundedSender, +} + +#[actix::test] +async fn rejected_log_fails_gateway_readiness() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-rejected-log"); + let gateway = EvmChainGateway::setup_with_readiness(&bus); + + gateway + .addr() + .send(InterfoldEvmEvent::Rejected(EvmLogRejected::new( + CorrelationId::new(), + 1, + "malformed historical log", + ))) + .await?; + + let error = gateway.wait_until_live().await.unwrap_err(); + assert!(error.to_string().contains("malformed historical log")); + Ok(()) +} + +impl Actor for SyncEventCollector { + type Context = actix::Context; +} + +impl Handler for SyncEventCollector { + type Result = (); + fn handle(&mut self, msg: HistoricalEvmEventsReceived, _: &mut Self::Context) { + let _ = self.tx.send(msg); + } +} + +#[actix::test] +async fn test_evm_chain_gateway() -> Result<()> { + let _foo = tracing::subscriber::set_default( + fmt() + .with_env_filter(EnvFilter::new("info")) + .with_test_writer() + .finish(), + ); + + let system = EventSystem::new().with_fresh_bus(); + let bus: BusHandle = system.handle()?.enable("test"); + + let history_collector = bus.history(); + + let (tx, mut rx) = mpsc::unbounded_channel(); + let collector = SyncEventCollector { tx }.start(); + + let gateway = EvmChainGateway::setup_with_readiness(&bus); + let addr = gateway.addr(); + + let chain_id = 1u64; + + // HistoricalEvmSyncStart: Init -> ForwardToSyncActor + let mut evm_config = EvmEventConfig::new(); + evm_config.insert(chain_id, EvmEventConfigChain::new(0)); + bus.publish_without_context(HistoricalEvmSyncStart::new(collector.clone(), evm_config)) + .unwrap(); + + // Send EVM event while forwarding - should reach collector + let evm_event = EvmEvent::new( + CorrelationId::new(), + TestEvent::new("Before Complete", 1).into(), + 100, + 12345, + chain_id, + ); + + // This will actually arrive earlier than HistoricalEvmSyncStart but aught to be buffered + addr.send(InterfoldEvmEvent::Event(evm_event)).await?; + + // HistoricalSyncComplete: ForwardToSyncActor -> BufferUntilLive + addr.send(InterfoldEvmEvent::HistoricalSyncComplete( + HistoricalSyncComplete::new(chain_id, None), + )) + .await?; + + // Normal Synchronizer will take this and wait for other events before flushing events to + // the bus here we simulate it + let received = rx.recv().await.unwrap(); + for event in received.events { + bus.naked_dispatch(event); + } + + // Send EVM event while buffering - should be buffered (not received) + let buffered_event = EvmEvent::new( + CorrelationId::new(), + TestEvent::new("Before SyncEnded", 2).into(), + 101, + 12346, + chain_id, + ); + addr.send(InterfoldEvmEvent::Event(buffered_event)).await?; + + // The Synchronizer will publish the SyncEnded event when it has all the information it needs + // and has published everything to the bus + bus.publish_without_context(SyncEnded::new())?; + gateway.wait_until_live().await?; + + let after_event = EvmEvent::new( + CorrelationId::new(), + TestEvent::new("After SyncEnded", 2).into(), + 101, + 12346, + chain_id, + ); + + addr.send(InterfoldEvmEvent::Event(after_event)).await?; + + let full = history_collector.send(TakeEvents::new(5)).await?; + + let test_events: Vec = full + .events + .iter() + .filter_map(|e| { + if let InterfoldEventData::TestEvent(TestEvent { msg, .. }) = e.get_data() { + Some(msg.to_string()) + } else { + None + } + }) + .collect(); + + assert_eq!( + test_events, + vec!["Before Complete", "Before SyncEnded", "After SyncEnded"] + ); + + let event_types: Vec = full.events.iter().map(|e| e.event_type()).collect(); + + assert_eq!( + event_types, + vec![ + "HistoricalEvmSyncStart", + "TestEvent", + "SyncEnded", + "TestEvent", + "TestEvent" + ] + ); + Ok(()) +} + +#[actix::test] +async fn overflow_emits_actionable_error_stops_and_fails_readiness() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus: BusHandle = system.handle()?.enable("test-overflow"); + let errors = bus.errors(); + let gateway = EvmChainGateway::setup_with_readiness_and_limit(&bus, 1); + let addr = gateway.addr(); + + for entropy in [1, 2] { + let event = EvmEvent::new( + CorrelationId::new(), + TestEvent::new("overflow", entropy).into(), + 100, + u128::from(entropy), + 1, + ); + addr.send(InterfoldEvmEvent::Event(event)).await?; + } + + let startup_error = gateway + .wait_until_live() + .await + .expect_err("overflow must fail gateway readiness") + .to_string(); + assert!(startup_error.contains("Init buffer reached its limit of 1 events")); + assert!(startup_error.contains("will not process further chain events")); + assert!(startup_error.contains("restart the node to replay chain history")); + + let received = errors.send(TakeEvents::new(1)).await?; + assert!(!received.timed_out, "overflow error should be observable"); + let InterfoldEventData::InterfoldError(error) = received.events[0].get_data() else { + panic!("expected an InterfoldError event"); + }; + assert!(error + .message + .contains("Init buffer reached its limit of 1 events")); + assert!(error.message.contains("snapshot/deploy block")); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while addr.connected() { + tokio::task::yield_now().await; + } + }) + .await + .context("overflowed gateway did not stop")?; + assert!(!addr.connected(), "overflowed gateway must stop"); + Ok(()) +} diff --git a/crates/evm/src/actors/evm_hub.rs b/crates/evm/src/chain_hub.rs similarity index 100% rename from crates/evm/src/actors/evm_hub.rs rename to crates/evm/src/chain_hub.rs diff --git a/crates/evm/src/chain_reader/actor.rs b/crates/evm/src/chain_reader/actor.rs new file mode 100644 index 0000000000..79d894ff86 --- /dev/null +++ b/crates/evm/src/chain_reader/actor.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::adapters::log_fetcher::{ + backfill_to_head, fetch_logs_chunked, process_live_log, TimestampTracker, +}; +use crate::domain::backoff::Backoff; +use crate::helpers::{EthProvider, ProviderFactory}; +use crate::messages::HistoricalSyncComplete; +use crate::messages::{EvmEventProcessor, InterfoldEvmEvent}; +use actix::prelude::*; +use alloy::eips::BlockNumberOrTag; +use alloy::primitives::B256; +use alloy::providers::Provider; +use alloy::rpc::types::Filter; +use alloy_primitives::Address; +use anyhow::anyhow; +use e3_events::{ + BusHandle, EType, ErrorDispatcher, Event, EventId, InterfoldEvent, InterfoldEventData, +}; +use e3_events::{EventSubscriber, EventType}; +use e3_utils::{retry_with_backoff, RetryError, MAILBOX_LIMIT}; +use futures_util::stream::StreamExt; +use std::collections::{HashMap, HashSet}; +use std::time::Duration; +use tokio::select; +use tokio::sync::oneshot; +use tracing::{error, info, instrument, warn}; + +#[path = "effects/mod.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +use effects::stream::stream_from_evm; + +const MAX_RECONNECT_DELAY_SECS: u64 = 60; +/// Maximum attempts to recreate a provider via the factory before adding an +/// extra outer delay. +const PROVIDER_RECREATE_MAX_ATTEMPTS: u32 = 3; +/// Initial delay (ms) between provider-recreation attempts. +const PROVIDER_RECREATE_INITIAL_DELAY_MS: u64 = 2000; +/// Consecutive failures before we assume the provider is dead and recreate it. +const MAX_RETRIES_BEFORE_RECREATE: u32 = 3; +/// Polling is required even while the subscription is quiet: a log becomes confirmed because later +/// blocks arrive, and those blocks need not contain any matching contract event. +const CONFIRMED_BACKFILL_INTERVAL_SECS: u64 = 5; + +#[derive(Default, serde::Serialize, serde::Deserialize, Clone)] +pub struct EvmReadInterfaceState { + pub ids: HashSet, + pub last_block: Option, +} + +#[derive(Clone, Default)] +pub struct Filters { + historical: Filter, + current: Filter, + start_block: u64, + /// Number of confirmations required before a log is ingested. `0` (default) + /// reads to the chain head exactly as before; a positive value clamps the + /// historical and backfill heads to make ingestion reorg-safe. + confirmations: u64, +} + +impl Filters { + pub fn new(addresses: Vec

, start_block: u64) -> Self { + let historical = Filter::new() + .address(addresses.clone()) + .from_block(start_block); + let current = Filter::new() + .address(addresses) + .from_block(BlockNumberOrTag::Latest); + + Self { + historical, + current, + start_block, + confirmations: 0, + } + } + + /// Builder: require `confirmations` block confirmations before ingestion. + pub fn with_confirmations(mut self, confirmations: u64) -> Self { + self.confirmations = confirmations; + self + } + + /// The configured confirmation depth (0 = read to head). + pub fn confirmations(&self) -> u64 { + self.confirmations + } + + pub fn from_routing_table(table: &HashMap, start_block: u64) -> Self { + let addresses: Vec
= table.keys().cloned().collect(); + Self::new(addresses, start_block) + } +} + +/// Connects to Interfold.sol converting EVM events to InterfoldEvents +pub struct EvmReadInterface

{ + /// The alloy provider + provider: Option>, + /// Optional factory to recreate the provider when the transport dies + provider_factory: Option>, + /// A shutdown receiver to listen to for shutdown signals sent to the loop this is only used + /// internally. You should send the Shutdown signal to the reader directly or via the EventBus + shutdown_rx: Option>, + /// The sender for the shutdown signal this is only used internally + shutdown_tx: Option>, + /// Processor to forward events + next: EvmEventProcessor, + /// Event bus for error propagation only + bus: BusHandle, + /// Filters to configure when to seek from + filters: Filters, +} + +impl EvmReadInterface

{ + pub fn setup( + provider: &EthProvider

, + next: impl Into, + bus: &BusHandle, + filters: Filters, + ) -> Addr { + Self::setup_with_factory(provider, None, next, bus, filters) + } + + pub fn setup_with_factory( + provider: &EthProvider

, + provider_factory: Option>, + next: impl Into, + bus: &BusHandle, + filters: Filters, + ) -> Addr { + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + let reader = Self { + provider: Some(provider.clone()), + provider_factory, + shutdown_rx: Some(shutdown_rx), + shutdown_tx: Some(shutdown_tx), + next: next.into(), + bus: bus.clone(), + filters, + }; + + let addr = reader.start(); + bus.subscribe(EventType::Shutdown, addr.clone().into()); + addr + } +} diff --git a/crates/evm/src/domain/backoff.rs b/crates/evm/src/chain_reader/backoff.rs similarity index 100% rename from crates/evm/src/domain/backoff.rs rename to crates/evm/src/chain_reader/backoff.rs diff --git a/crates/evm/src/chain_reader/effects/mod.rs b/crates/evm/src/chain_reader/effects/mod.rs new file mode 100644 index 0000000000..fe46fa18f8 --- /dev/null +++ b/crates/evm/src/chain_reader/effects/mod.rs @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Provider recovery and bounded EVM stream execution. + +use super::*; + +mod provider_recovery; +pub(super) mod stream; diff --git a/crates/evm/src/chain_reader/effects/provider_recovery.rs b/crates/evm/src/chain_reader/effects/provider_recovery.rs new file mode 100644 index 0000000000..2ea14f788f --- /dev/null +++ b/crates/evm/src/chain_reader/effects/provider_recovery.rs @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Provider recreation and shutdown-aware backoff. + +use super::*; + +pub(super) async fn sleep_or_shutdown( + duration: Duration, + shutdown: &mut oneshot::Receiver<()>, +) -> bool { + select! { + _ = tokio::time::sleep(duration) => false, + _ = &mut *shutdown => { + info!("Shutdown signal received during backoff"); + true + } + } +} + +async fn recreate_provider( + factory: &ProviderFactory

, + shutdown: &mut oneshot::Receiver<()>, + chain_id: u64, + backoff: &mut Backoff, +) -> Option> { + loop { + if shutdown.try_recv().is_ok() { + return None; + } + + let delay = backoff.next_delay(); + warn!( + chain_id, + delay_secs = delay.as_secs(), + "Waiting before provider recreation attempt" + ); + if sleep_or_shutdown(delay, shutdown).await { + return None; + } + + let factory_clone = factory.clone(); + let result = retry_with_backoff( + || { + let f = factory_clone.clone(); + async move { + let provider = f().await.map_err(|e| { + warn!(chain_id, error = %e, "Factory failed to create provider"); + RetryError::Retry(e) + })?; + + // Health check: verify the new transport is actually alive + provider.provider().get_block_number().await.map_err(|e| { + warn!(chain_id, error = %e, "New provider failed health check"); + RetryError::Retry(anyhow!("Health check failed: {}", e)) + })?; + + Ok(provider) + } + }, + PROVIDER_RECREATE_MAX_ATTEMPTS, + PROVIDER_RECREATE_INITIAL_DELAY_MS, + ) + .await; + + match result { + Ok(new_provider) => { + let new_chain_id = new_provider.chain_id(); + if new_chain_id != chain_id { + error!( + chain_id, + new_chain_id, "Recreated provider is on wrong chain — fatal" + ); + return None; + } + info!(chain_id, "Provider recreated and verified"); + backoff.reset(); + return Some(new_provider); + } + Err(e) => { + error!( + chain_id, + error = %e, + "All provider recreation attempts failed, will retry with longer backoff" + ); + continue; + } + } + } +} + +pub(super) async fn get_new_provider_or_exit( + factory: &Option>, + shutdown: &mut oneshot::Receiver<()>, + chain_id: u64, + backoff: &mut Backoff, + bus: &BusHandle, +) -> Option> { + let Some(factory) = factory else { + error!( + chain_id, + "Transport died and no provider factory configured" + ); + bus.err( + EType::Evm, + anyhow!("Transport died and no provider factory configured"), + ); + return None; + }; + let result = recreate_provider(factory, shutdown, chain_id, backoff).await; + if result.is_none() && shutdown.try_recv().is_err() { + bus.err( + EType::Evm, + anyhow!("Provider recreation failed for chain {}", chain_id), + ); + } + result +} diff --git a/crates/evm/src/chain_reader/effects/stream.rs b/crates/evm/src/chain_reader/effects/stream.rs new file mode 100644 index 0000000000..33db020b7f --- /dev/null +++ b/crates/evm/src/chain_reader/effects/stream.rs @@ -0,0 +1,236 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Historical catch-up and confirmed live-log ingestion loop. + +use super::provider_recovery::{get_new_provider_or_exit, sleep_or_shutdown}; +use super::*; + +#[instrument(name = "evm_interface", skip_all)] +pub(in crate::actors::evm_read_interface) async fn stream_from_evm< + P: Provider + Clone + 'static, +>( + provider: EthProvider

, + provider_factory: Option>, + next: EvmEventProcessor, + mut shutdown: oneshot::Receiver<()>, + bus: &BusHandle, + filters: Filters, +) { + let chain_id = provider.chain_id(); + let mut timestamp_tracker = TimestampTracker::new(); + let mut backoff = Backoff::new(MAX_RECONNECT_DELAY_SECS); + + // ── Phase 1: Historical sync (must succeed, fatal on failure) ── + + let latest_block = match provider.provider().get_block_number().await { + Ok(bn) => crate::domain::reorg::confirmed_head(bn, filters.confirmations()), + Err(e) => { + error!(chain_id, error = %e, "Failed to get latest block number"); + bus.err(EType::Evm, anyhow!(e)); + return; + } + }; + + let last_id = match fetch_logs_chunked( + provider.provider(), + &filters.historical, + filters.start_block, + latest_block, + chain_id, + &next, + &mut timestamp_tracker, + ) + .await + { + Ok(id) => { + info!(chain_id, "Historical sync succeeded"); + id + } + Err(e) => { + error!(chain_id, error = %e, "Failed to fetch historical events — node cannot operate without full state, exiting"); + bus.err(EType::Evm, anyhow!(e)); + return; + } + }; + + next.do_send(InterfoldEvmEvent::HistoricalSyncComplete( + HistoricalSyncComplete::new(chain_id, last_id), + )); + + // ── Phase 2: Live event loop with provider lifecycle management ── + // + // Single flat loop: backfill → subscribe → consume stream → repeat. + // On transport death, immediately recreate the provider. + // On transient errors, retry with exponential backoff. + + let mut last_block = latest_block; + let mut current_provider = provider; + let mut consecutive_failures: u32 = 0; + + loop { + // Step 1: Backfill any blocks missed since last_block + match backfill_to_head( + current_provider.provider(), + &filters.current, + chain_id, + &next, + &mut timestamp_tracker, + &mut last_block, + filters.confirmations(), + ) + .await + { + Ok(_) => { + backoff.reset(); + consecutive_failures = 0; + } + Err(e) => { + consecutive_failures += 1; + warn!(chain_id, error = %e, consecutive_failures, "Backfill failed"); + if consecutive_failures >= MAX_RETRIES_BEFORE_RECREATE { + let Some(p) = get_new_provider_or_exit( + &provider_factory, + &mut shutdown, + chain_id, + &mut backoff, + bus, + ) + .await + else { + return; + }; + current_provider = p; + consecutive_failures = 0; + continue; + } + if sleep_or_shutdown(backoff.next_delay(), &mut shutdown).await { + return; + } + continue; + } + } + + // Step 2: Subscribe to live events + let sub_result = current_provider + .provider() + .subscribe_logs(&filters.current) + .await + .map_err(|e| anyhow!("{}", e)); + + match sub_result { + Ok(subscription) => { + backoff.reset(); + consecutive_failures = 0; + let sub_id: B256 = *subscription.local_id(); + let mut stream = subscription.into_stream(); + let mut confirmation_poll = + tokio::time::interval(Duration::from_secs(CONFIRMED_BACKFILL_INTERVAL_SECS)); + confirmation_poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + // Tokio intervals tick immediately once. The outer loop already backfilled, so + // consume that tick and wait for the configured period before querying again. + confirmation_poll.tick().await; + info!(chain_id, "Live event subscription active"); + + loop { + select! { + maybe_log = stream.next() => { + match maybe_log { + Some(log) => { + if let Err(error) = process_live_log( + current_provider.provider(), log, chain_id, &next, + &mut timestamp_tracker, &mut last_block, + filters.confirmations(), + ).await { + consecutive_failures += 1; + warn!( + chain_id, + error = %error, + consecutive_failures, + "Live log rejected; reconnecting for canonical backfill" + ); + break; + } + } + None => { + // Stream ended (server-side close, idle timeout, etc.) + consecutive_failures += 1; + warn!(chain_id, consecutive_failures, "Live event stream ended, will reconnect"); + break; + } + } + } + _ = confirmation_poll.tick(), if filters.confirmations() > 0 => { + if let Err(error) = backfill_to_head( + current_provider.provider(), + &filters.current, + chain_id, + &next, + &mut timestamp_tracker, + &mut last_block, + filters.confirmations(), + ).await { + consecutive_failures += 1; + warn!( + chain_id, + error = %error, + consecutive_failures, + "Confirmed live-log backfill failed; reconnecting" + ); + break; + } + } + _ = &mut shutdown => { + info!("Shutdown signal received, stopping EVM stream"); + let _ = current_provider.provider().unsubscribe(sub_id).await; + return; + } + } + } + + if consecutive_failures >= MAX_RETRIES_BEFORE_RECREATE { + let Some(p) = get_new_provider_or_exit( + &provider_factory, + &mut shutdown, + chain_id, + &mut backoff, + bus, + ) + .await + else { + return; + }; + current_provider = p; + consecutive_failures = 0; + } else if consecutive_failures > 0 + && sleep_or_shutdown(backoff.next_delay(), &mut shutdown).await + { + return; + } + } + Err(e) => { + consecutive_failures += 1; + error!(chain_id, error = %e, consecutive_failures, "Failed to subscribe to live events"); + if consecutive_failures >= MAX_RETRIES_BEFORE_RECREATE { + let Some(p) = get_new_provider_or_exit( + &provider_factory, + &mut shutdown, + chain_id, + &mut backoff, + bus, + ) + .await + else { + return; + }; + current_provider = p; + consecutive_failures = 0; + } else { + bus.err(EType::Evm, anyhow!("{}", e)); + if sleep_or_shutdown(backoff.next_delay(), &mut shutdown).await { + return; + } + } + } + } + } +} diff --git a/crates/evm/src/chain_reader/handlers.rs b/crates/evm/src/chain_reader/handlers.rs new file mode 100644 index 0000000000..b4c45ba16e --- /dev/null +++ b/crates/evm/src/chain_reader/handlers.rs @@ -0,0 +1,47 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Mailbox entry points and lifecycle hooks. + +use super::*; + +impl Actor for EvmReadInterface

{ + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + + let bus = self.bus.clone(); + let next = self.next.clone(); + let filters = self.filters.clone(); + let provider_factory = self.provider_factory.take(); + + let Some(provider) = self.provider.take() else { + error!("Could not start event reader as provider has already been used."); + return; + }; + + let Some(shutdown) = self.shutdown_rx.take() else { + bus.err(EType::Evm, anyhow!("shutdown already called")); + return; + }; + + ctx.spawn( + async move { + stream_from_evm(provider, provider_factory, next, shutdown, &bus, filters).await + } + .into_actor(self), + ); + } +} + +impl Handler for EvmReadInterface

{ + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { + if let InterfoldEventData::Shutdown(_) = msg.into_data() { + if let Some(shutdown) = self.shutdown_tx.take() { + let _ = shutdown.send(()); + } + } + } +} diff --git a/crates/evm/src/domain/log_timestamp.rs b/crates/evm/src/chain_reader/log_timestamp.rs similarity index 100% rename from crates/evm/src/domain/log_timestamp.rs rename to crates/evm/src/chain_reader/log_timestamp.rs diff --git a/crates/evm/src/domain/reorg.rs b/crates/evm/src/chain_reader/reorg.rs similarity index 100% rename from crates/evm/src/domain/reorg.rs rename to crates/evm/src/chain_reader/reorg.rs diff --git a/crates/evm/src/actors/sync_start_extractor.rs b/crates/evm/src/chain_sync/start_extractor.rs similarity index 100% rename from crates/evm/src/actors/sync_start_extractor.rs rename to crates/evm/src/chain_sync/start_extractor.rs diff --git a/crates/evm/src/ciphernode_registry/actor.rs b/crates/evm/src/ciphernode_registry/actor.rs new file mode 100644 index 0000000000..38d3a39943 --- /dev/null +++ b/crates/evm/src/ciphernode_registry/actor.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Ciphernode registry EVM boundary. +//! +//! The actor owns subscription, routing, and in-flight submission state. Contract +//! reads and transactions live in `transactions`; message handling lives in +//! `handlers`. + +use crate::actors::evm_parser::EvmParser; +use crate::contracts::ICiphernodeRegistry; +use crate::domain::ciphernode_registry_events::extractor; +use crate::domain::error_decoder::{decode_error_from_str, format_evm_error}; +use crate::helpers::{encode_zk_proof, send_tx_with_retry, transaction_nonce_guard, EthProvider}; +use crate::messages::{EvmEventProcessor, InterfoldEvmEvent}; +use actix::prelude::*; +use alloy::{ + primitives::{Address, Bytes, B256, U256}, + providers::{Provider, WalletProvider}, + rpc::types::TransactionReceipt, +}; +use anyhow::Result; +use e3_events::{ + prelude::*, AggregatorChanged, BusHandle, CommitteeFinalizeRequested, E3RequestComplete, E3id, + EType, EffectsEnabled, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, Proof, + PublicKeyAggregated, Shutdown, TicketGenerated, TicketId, +}; +use e3_utils::{require_successful_receipt, ArcBytes, NotifySync, MAILBOX_LIMIT}; +use std::collections::{HashMap, HashSet}; +use tracing::{error, info}; + +#[path = "effects.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +#[allow(unused_imports)] +pub use effects::{ + fetch_accusation_vote_validity, fetch_dkg_fold_attestation_verifier, + finalize_committee_on_registry, publish_committee_to_registry, submit_ticket_to_registry, +}; + +/// Connects to CiphernodeRegistry.sol converting EVM events to InterfoldEvents. +pub struct CiphernodeRegistrySolReader; + +impl CiphernodeRegistrySolReader { + pub fn setup(next: &EvmEventProcessor) -> Addr { + EvmParser::new(next, extractor).start() + } +} + +/// Writer for publishing committees to CiphernodeRegistry. +pub struct CiphernodeRegistrySolWriter

{ + provider: EthProvider

, + contract_address: Address, + bus: BusHandle, + effects_enabled: bool, + active_aggregators: HashMap, + /// Session-local concurrency guard. On-chain preflight is the durable + /// cross-restart idempotency boundary. + submitting: HashSet, +} + +impl CiphernodeRegistrySolWriter

{ + pub fn new( + bus: &BusHandle, + provider: EthProvider

, + contract_address: Address, + ) -> Result { + Ok(Self { + provider, + contract_address, + bus: bus.clone(), + effects_enabled: false, + active_aggregators: HashMap::new(), + submitting: HashSet::new(), + }) + } + + pub fn attach(bus: &BusHandle, provider: EthProvider

, contract_address: Address) { + let addr = CiphernodeRegistrySolWriter::new(bus, provider, contract_address) + .expect("failed to create CiphernodeRegistrySolWriter") + .start(); + + bus.subscribe_all( + &[ + EventType::EffectsEnabled, + EventType::AggregatorChanged, + EventType::PublicKeyAggregated, + EventType::CommitteeFinalizeRequested, + EventType::TicketGenerated, + EventType::E3RequestComplete, + EventType::Shutdown, + ], + addr.into(), + ); + } + + fn is_active_aggregator_for(&self, e3_id: &E3id) -> bool { + self.active_aggregators.get(e3_id).copied().unwrap_or(false) + } +} + +/// Wrapper for a reader and writer. +pub struct CiphernodeRegistrySol; + +impl CiphernodeRegistrySol { + pub fn attach(processor: &Recipient) -> Addr { + CiphernodeRegistrySolReader::setup(processor) + } + + pub fn attach_writer

(bus: &BusHandle, provider: EthProvider

, contract_address: Address) + where + P: Provider + WalletProvider + Clone + 'static, + { + CiphernodeRegistrySolWriter::attach(bus, provider, contract_address); + } +} diff --git a/crates/evm/src/ciphernode_registry/effects.rs b/crates/evm/src/ciphernode_registry/effects.rs new file mode 100644 index 0000000000..71b8e15896 --- /dev/null +++ b/crates/evm/src/ciphernode_registry/effects.rs @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Idempotency preflights and CiphernodeRegistry contract effects. + +use super::*; + +pub async fn submit_ticket_to_registry( + provider: EthProvider

, + contract_address: Address, + e3_id: E3id, + ticket_number: u64, +) -> Result { + let e3_id_u256: U256 = e3_id.try_into()?; + let ticket_number_u256 = U256::from(ticket_number); + + send_tx_with_retry("submitTicket", &["CommitteeNotRequested"], || { + let provider = provider.clone(); + async move { + info!("Calling: contract.submitTicket(..)"); + let _nonce_guard = transaction_nonce_guard(&provider).await; + let from_address = provider.provider().default_signer_address(); + let current_nonce = provider + .provider() + .get_transaction_count(from_address) + .pending() + .await?; + let contract = ICiphernodeRegistry::new(contract_address, provider.provider()); + let builder = contract + .submitTicket(e3_id_u256, ticket_number_u256) + .nonce(current_nonce); + let pending = builder.send().await?; + drop(_nonce_guard); + let receipt = pending.get_receipt().await?; + require_successful_receipt("submit ticket", &receipt)?; + Ok(receipt) + } + }) + .await +} + +pub async fn finalize_committee_on_registry( + provider: EthProvider

, + contract_address: Address, + e3_id: E3id, +) -> Result { + let e3_id_u256: U256 = e3_id.try_into()?; + + send_tx_with_retry( + "finalizeCommittee", + &[ + "SubmissionWindowNotClosed", + "CommitteeNotRequested", + "ThresholdNotMet", + ], + || { + let provider = provider.clone(); + async move { + info!("Calling: contract.finalizeCommittee(..)"); + let _nonce_guard = transaction_nonce_guard(&provider).await; + let from_address = provider.provider().default_signer_address(); + let current_nonce = provider + .provider() + .get_transaction_count(from_address) + .pending() + .await?; + let contract = ICiphernodeRegistry::new(contract_address, provider.provider()); + let builder = contract.finalizeCommittee(e3_id_u256).nonce(current_nonce); + let pending = builder.send().await?; + drop(_nonce_guard); + let receipt = pending.get_receipt().await?; + require_successful_receipt("finalize committee", &receipt)?; + Ok(receipt) + } + }, + ) + .await +} + +pub(in crate::actors::ciphernode_registry_sol) async fn should_finalize_committee< + P: Provider + WalletProvider + Clone + 'static, +>( + provider: EthProvider

, + contract_address: Address, + e3_id: E3id, +) -> Result { + let e3_id_u256: U256 = e3_id.try_into()?; + let contract = ICiphernodeRegistry::new(contract_address, provider.provider()); + if contract.isOpen(e3_id_u256).call().await? { + return Ok(false); + } + + match contract.finalizeCommittee(e3_id_u256).call().await { + Ok(_) => Ok(true), + Err(err) => { + let err = anyhow::Error::from(err); + let decoded = decode_error_from_str(&format!("{err:?}")); + + if decoded.as_deref().is_some_and(|message| { + message.contains("CommitteeAlreadyFinalized") + || message.contains("CommitteeNotRequested") + || message.contains("SubmissionWindowNotClosed") + || message.contains("ThresholdNotMet") + }) { + return Ok(false); + } + + Err(err) + } + } +} + +pub(in crate::actors::ciphernode_registry_sol) async fn should_publish_committee< + P: Provider + WalletProvider + Clone + 'static, +>( + provider: EthProvider

, + contract_address: Address, + e3_id: E3id, +) -> Result { + let e3_id_u256: U256 = e3_id.try_into()?; + let contract = ICiphernodeRegistry::new(contract_address, provider.provider()); + match contract.committeePublicKey(e3_id_u256).call().await { + Ok(_) => Ok(false), + Err(err) => { + let err = anyhow::Error::from(err); + let decoded = decode_error_from_str(&format!("{err:?}")); + + if decoded + .as_deref() + .is_some_and(|message| message.contains("CommitteeNotPublished")) + { + return Ok(true); + } + + Err(err) + } + } +} + +pub async fn publish_committee_to_registry( + provider: EthProvider

, + contract_address: Address, + e3_id: E3id, + public_key: ArcBytes, + pk_commitment: [u8; 32], + dkg_aggregator_proof: Option<&Proof>, + dkg_attestation_bundle: Option<&[u8]>, +) -> Result { + let e3_id_u256: U256 = e3_id.try_into()?; + let public_key_bytes = Bytes::from(public_key.extract_bytes()); + let pk_commitment_b256 = B256::from(pk_commitment); + + // `proof` is empty when `proofAggregationEnabled = false`; the contract + // trusts `pk_commitment` directly in that case. + let proof: Bytes = match dkg_aggregator_proof { + Some(p) => encode_zk_proof(p)?, + None => Bytes::new(), + }; + let attestation_bundle: Bytes = match dkg_attestation_bundle { + Some(b) => Bytes::copy_from_slice(b), + None => Bytes::new(), + }; + + // RPC may not have synced finalization yet + send_tx_with_retry("publishCommittee", &["CommitteeNotFinalized"], || { + let provider = provider.clone(); + let public_key_bytes = public_key_bytes.clone(); + let proof = proof.clone(); + let attestation_bundle = attestation_bundle.clone(); + async move { + info!("Calling: contract.publishCommittee(..)"); + let _nonce_guard = transaction_nonce_guard(&provider).await; + let from_address = provider.provider().default_signer_address(); + let current_nonce = provider + .provider() + .get_transaction_count(from_address) + .pending() + .await?; + let contract = ICiphernodeRegistry::new(contract_address, provider.provider()); + let builder = contract + .publishCommittee( + e3_id_u256, + public_key_bytes, + pk_commitment_b256, + proof, + attestation_bundle, + ) + .nonce(current_nonce); + let pending = builder.send().await?; + drop(_nonce_guard); + let receipt = pending.get_receipt().await?; + require_successful_receipt("publish committee", &receipt)?; + Ok(receipt) + } + }) + .await +} + +/// Read `CiphernodeRegistry.dkgFoldAttestationVerifier()` (EIP-712 verifying contract for fold attestations). +pub async fn fetch_dkg_fold_attestation_verifier( + provider: &P, + registry_address: Address, +) -> Result> { + let contract = ICiphernodeRegistry::new(registry_address, provider); + let verifier = contract.dkgFoldAttestationVerifier().call().await?; + if verifier == Address::ZERO { + Ok(None) + } else { + Ok(Some(verifier)) + } +} + +/// Read `CiphernodeRegistry.accusationVoteValidity()` — registry-wide off-chain +/// freshness window (seconds) accusers stamp on `AccusationVote.deadline`. +/// Returns the raw `uint256` as `U256`; callers decide how to clamp it to +/// their own arithmetic type. `Ok(None)` is reserved for the case where the +/// registry has been governance-disabled (`accusationVoteValidity = 0`) so +/// the caller can short-circuit without producing votes that will never +/// verify on chain. +pub async fn fetch_accusation_vote_validity( + provider: &P, + registry_address: Address, +) -> Result> { + let contract = ICiphernodeRegistry::new(registry_address, provider); + let validity = contract.accusationVoteValidity().call().await?; + if validity.is_zero() { + Ok(None) + } else { + Ok(Some(validity)) + } +} diff --git a/crates/evm/src/domain/ciphernode_registry_events.rs b/crates/evm/src/ciphernode_registry/events.rs similarity index 100% rename from crates/evm/src/domain/ciphernode_registry_events.rs rename to crates/evm/src/ciphernode_registry/events.rs diff --git a/crates/evm/src/ciphernode_registry/handlers.rs b/crates/evm/src/ciphernode_registry/handlers.rs new file mode 100644 index 0000000000..0b91ecea54 --- /dev/null +++ b/crates/evm/src/ciphernode_registry/handlers.rs @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Actix routing and lifecycle handlers for the registry writer. + +use super::effects::*; +use super::*; + +impl Actor for CiphernodeRegistrySolWriter

{ + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } +} + +impl Handler + for CiphernodeRegistrySolWriter

+{ + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + match msg.into_data() { + InterfoldEventData::EffectsEnabled(data) => self.notify_sync(ctx, data), + InterfoldEventData::AggregatorChanged(data) => self.notify_sync(ctx, data), + InterfoldEventData::PublicKeyAggregated(data) => { + // Only publish if the src and destination chains match + if self.provider.chain_id() == data.e3_id.chain_id() { + ctx.notify(data); + } + } + InterfoldEventData::CommitteeFinalizeRequested(data) => { + if self.provider.chain_id() == data.e3_id.chain_id() { + ctx.notify(data); + } + } + InterfoldEventData::TicketGenerated(data) => { + // Submit ticket if chain matches + if self.provider.chain_id() == data.e3_id.chain_id() { + ctx.notify(data); + } + } + InterfoldEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), + InterfoldEventData::Shutdown(data) => self.notify_sync(ctx, data), + _ => (), + } + } +} + +impl Handler + for CiphernodeRegistrySolWriter

+{ + type Result = (); + + fn handle(&mut self, _: EffectsEnabled, _: &mut Self::Context) -> Self::Result { + self.effects_enabled = true; + } +} + +impl Handler + for CiphernodeRegistrySolWriter

+{ + type Result = (); + + fn handle(&mut self, msg: AggregatorChanged, _: &mut Self::Context) -> Self::Result { + self.active_aggregators.insert(msg.e3_id, msg.is_aggregator); + } +} + +impl Handler + for CiphernodeRegistrySolWriter

+{ + type Result = (); + + fn handle(&mut self, msg: E3RequestComplete, _: &mut Self::Context) -> Self::Result { + self.active_aggregators.remove(&msg.e3_id); + self.submitting.remove(&msg.e3_id); + } +} + +impl Handler + for CiphernodeRegistrySolWriter

+{ + type Result = ResponseFuture<()>; + + fn handle(&mut self, msg: TicketGenerated, _: &mut Self::Context) -> Self::Result { + if !self.effects_enabled { + return Box::pin(async {}); + } + + match msg.ticket_id { + TicketId::Score(ticket_id) => { + info!( + "Score sortition ticket generated for E3 {:?}, submitting to contract", + msg.e3_id + ); + + let e3_id = msg.e3_id.clone(); + let contract_address = self.contract_address; + let provider = self.provider.clone(); + let bus = self.bus.clone(); + + Box::pin(async move { + info!("Submitting ticket {} for E3 {:?}", ticket_id, e3_id); + + let result = + submit_ticket_to_registry(provider, contract_address, e3_id, ticket_id) + .await; + match result { + Ok(receipt) => { + info!(tx=%receipt.transaction_hash, "Ticket submitted to registry"); + } + Err(err) => { + error!("Failed to submit ticket: {}", format_evm_error(&err)); + bus.err(EType::Evm, err); + } + } + }) + } + } + } +} + +impl Handler + for CiphernodeRegistrySolWriter

+{ + type Result = ResponseFuture<()>; + + fn handle(&mut self, msg: CommitteeFinalizeRequested, _: &mut Self::Context) -> Self::Result { + if !self.effects_enabled { + return Box::pin(async {}); + } + + let e3_id = msg.e3_id.clone(); + let contract_address = self.contract_address; + let provider = self.provider.clone(); + let bus = self.bus.clone(); + + Box::pin(async move { + match should_finalize_committee(provider.clone(), contract_address, e3_id.clone()).await + { + Ok(false) => { + info!(e3_id = %e3_id, "Skipping finalizeCommittee; on-chain state is not finalizable"); + return; + } + Err(err) => { + error!( + "Failed to preflight finalizeCommittee: {}", + format_evm_error(&err) + ); + return; + } + Ok(true) => {} + } + + info!("Finalizing committee for E3 {:?}", e3_id); + + let result = finalize_committee_on_registry(provider, contract_address, e3_id).await; + match result { + Ok(receipt) => { + info!(tx=%receipt.transaction_hash, "Committee finalized on registry"); + } + Err(err) => { + error!("Failed to finalize committee: {}", format_evm_error(&err)); + bus.err(EType::Evm, err); + } + } + }) + } +} + +impl Handler + for CiphernodeRegistrySolWriter

+{ + type Result = ResponseFuture<()>; + + fn handle(&mut self, msg: PublicKeyAggregated, ctx: &mut Self::Context) -> Self::Result { + if !self.effects_enabled || !self.is_active_aggregator_for(&msg.e3_id) { + return Box::pin(async {}); + } + + // Don't fire a second on-chain submission for an E3 whose publishCommittee + // tx is already in flight (H13). The on-chain preflight below is still the + // authoritative idempotency guard across restarts. + if !self.submitting.insert(msg.e3_id.clone()) { + info!(e3_id = %msg.e3_id, "publishCommittee already in flight; skipping duplicate submission"); + return Box::pin(async {}); + } + + let e3_id = msg.e3_id.clone(); + let pubkey = msg.pubkey.clone(); + let pk_commitment = msg.pk_commitment; + let dkg_aggregator_proof = msg.dkg_aggregator_proof.clone(); + let dkg_attestation_bundle = msg.dkg_attestation_bundle.clone(); + let contract_address = self.contract_address; + let provider = self.provider.clone(); + let bus = self.bus.clone(); + let self_addr = ctx.address(); + + Box::pin(async move { + match should_publish_committee(provider.clone(), contract_address, e3_id.clone()).await + { + Ok(false) => { + info!(e3_id = %e3_id, "Skipping publishCommittee; committee public key already published"); + return; + } + Err(err) => { + error!( + "Failed to preflight publishCommittee: {}", + format_evm_error(&err) + ); + // Transient read failure: allow a later event to retry. + self_addr.do_send(ClearSubmitting(e3_id)); + return; + } + Ok(true) => {} + } + + let result = publish_committee_to_registry( + provider, + contract_address, + e3_id.clone(), + pubkey, + pk_commitment, + dkg_aggregator_proof.as_ref(), + dkg_attestation_bundle.as_ref().map(|b| b.as_ref()), + ) + .await; + match result { + Ok(receipt) => { + info!(tx=%receipt.transaction_hash, "Committee published to registry"); + } + Err(err) => { + error!("Failed to publish committee: {}", format_evm_error(&err)); + // Submission failed: clear the in-flight marker so a retry can proceed. + self_addr.do_send(ClearSubmitting(e3_id)); + bus.err(EType::Evm, err); + } + } + }) + } +} + +/// Internal message: clear the in-flight `publishCommittee` marker for an E3 so +/// a subsequent submission attempt is allowed after a failure (H13). +#[derive(Message)] +#[rtype(result = "()")] +struct ClearSubmitting(E3id); + +impl Handler + for CiphernodeRegistrySolWriter

+{ + type Result = (); + + fn handle(&mut self, msg: ClearSubmitting, _: &mut Self::Context) -> Self::Result { + self.submitting.remove(&msg.0); + } +} + +impl Handler + for CiphernodeRegistrySolWriter

+{ + type Result = (); + + fn handle(&mut self, _: Shutdown, ctx: &mut Self::Context) -> Self::Result { + ctx.stop(); + } +} diff --git a/crates/evm/src/domain/mod.rs b/crates/evm/src/domain.rs similarity index 62% rename from crates/evm/src/domain/mod.rs rename to crates/evm/src/domain.rs index 3928d41fdc..490b2084ae 100644 --- a/crates/evm/src/domain/mod.rs +++ b/crates/evm/src/domain.rs @@ -4,28 +4,43 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -//! Pure, synchronous, unit-testable business logic for the EVM integration. +//! Compatibility view of pure EVM integration logic stored by capability. //! //! Nothing in this module depends on the actix runtime, `BusHandle`, or actor //! addresses; the `actors` module wires these services into message-passing //! shells and performs the actual EVM/provider I/O. // `error_decoder` is part of the crate's public surface (`e3_evm::error_decoder`). +#[path = "event_decoding/error.rs"] pub mod error_decoder; +#[path = "slashing/evidence.rs"] pub(crate) mod attestation_evidence; +#[path = "chain_reader/backoff.rs"] pub(crate) mod backoff; +#[path = "bonding_registry/events.rs"] pub(crate) mod bonding_registry_events; +#[path = "chain_gateway/state.rs"] pub(crate) mod chain_sync_state; +#[path = "ciphernode_registry/events.rs"] pub(crate) mod ciphernode_registry_events; +#[path = "event_decoding/catalog.rs"] pub(crate) mod evm_event_catalog; +#[path = "event_decoding/observation.rs"] pub(crate) mod evm_log_observation; +#[path = "historical_order/workflow.rs"] pub(crate) mod historical_order_fixer; +#[path = "interfold/events.rs"] pub(crate) mod interfold_events; +#[path = "chain_reader/log_timestamp.rs"] pub(crate) mod log_timestamp; +#[path = "interfold_writing/workflow.rs"] pub(crate) mod plaintext_publication; +#[path = "chain_reader/reorg.rs"] pub(crate) mod reorg; +#[path = "slashing_writing/workflow.rs"] pub(crate) mod slash_submission; +#[path = "slashing/events.rs"] pub(crate) mod slashing_events; pub use attestation_evidence::encode_attestation_evidence; diff --git a/crates/evm/src/actors/evm_parser.rs b/crates/evm/src/event_decoding/actor.rs similarity index 100% rename from crates/evm/src/actors/evm_parser.rs rename to crates/evm/src/event_decoding/actor.rs diff --git a/crates/evm/src/domain/evm_event_catalog.rs b/crates/evm/src/event_decoding/catalog.rs similarity index 100% rename from crates/evm/src/domain/evm_event_catalog.rs rename to crates/evm/src/event_decoding/catalog.rs diff --git a/crates/evm/src/domain/error_decoder.rs b/crates/evm/src/event_decoding/error.rs similarity index 100% rename from crates/evm/src/domain/error_decoder.rs rename to crates/evm/src/event_decoding/error.rs diff --git a/crates/evm/src/domain/evm_log_observation.rs b/crates/evm/src/event_decoding/observation.rs similarity index 100% rename from crates/evm/src/domain/evm_log_observation.rs rename to crates/evm/src/event_decoding/observation.rs diff --git a/crates/evm/src/actors/evm_router.rs b/crates/evm/src/event_router.rs similarity index 100% rename from crates/evm/src/actors/evm_router.rs rename to crates/evm/src/event_router.rs diff --git a/crates/evm/src/actors/fix_historical_order.rs b/crates/evm/src/historical_order/actor.rs similarity index 100% rename from crates/evm/src/actors/fix_historical_order.rs rename to crates/evm/src/historical_order/actor.rs diff --git a/crates/evm/src/domain/historical_order_fixer.rs b/crates/evm/src/historical_order/workflow.rs similarity index 100% rename from crates/evm/src/domain/historical_order_fixer.rs rename to crates/evm/src/historical_order/workflow.rs diff --git a/crates/evm/src/domain/interfold_events.rs b/crates/evm/src/interfold/events.rs similarity index 100% rename from crates/evm/src/domain/interfold_events.rs rename to crates/evm/src/interfold/events.rs diff --git a/crates/evm/src/actors/interfold_sol_reader.rs b/crates/evm/src/interfold/reader.rs similarity index 100% rename from crates/evm/src/actors/interfold_sol_reader.rs rename to crates/evm/src/interfold/reader.rs diff --git a/crates/evm/src/interfold_writing/actor.rs b/crates/evm/src/interfold_writing/actor.rs new file mode 100644 index 0000000000..f8b831c6a8 --- /dev/null +++ b/crates/evm/src/interfold_writing/actor.rs @@ -0,0 +1,91 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Interfold contract publication boundary. + +use crate::contracts::IInterfold; +use crate::domain::error_decoder::format_evm_error; +use crate::domain::plaintext_publication::validate_plaintext_output; +use crate::helpers::{encode_zk_proof, transaction_nonce_guard, EthProvider}; +use crate::send_tx_with_retry; +use actix::prelude::*; +use alloy::{ + primitives::{Address, Bytes, U256}, + providers::{Provider, WalletProvider}, + rpc::types::TransactionReceipt, +}; +use anyhow::Result; +use e3_events::{ + prelude::*, AggregatorChanged, BusHandle, E3RequestComplete, E3Stage, E3StageChanged, E3id, + EType, EffectsEnabled, EventType, InterfoldEvent, InterfoldEventData, PlaintextAggregated, + Proof, Shutdown, +}; +use e3_utils::{require_successful_receipt, NotifySync, MAILBOX_LIMIT}; +use std::collections::{HashMap, HashSet}; +use tracing::info; + +#[path = "effects.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +/// Consumes events from the event bus and calls EVM methods on the Interfold contract. +pub struct InterfoldSolWriter

{ + provider: EthProvider

, + contract_address: Address, + bus: BusHandle, + effects_enabled: bool, + active_aggregators: HashMap, + /// Session-local concurrency guard. The contract preflight remains the + /// durable cross-restart idempotency boundary. + submitting: HashSet, +} + +impl InterfoldSolWriter

{ + pub fn new( + bus: &BusHandle, + provider: EthProvider

, + contract_address: Address, + ) -> Result { + Ok(Self { + provider, + contract_address, + bus: bus.clone(), + effects_enabled: false, + active_aggregators: HashMap::new(), + submitting: HashSet::new(), + }) + } + + pub fn attach(bus: &BusHandle, provider: EthProvider

, contract_address: Address) { + let addr = InterfoldSolWriter::new(bus, provider, contract_address) + .expect("failed to create InterfoldSolWriter") + .start(); + bus.subscribe_all( + &[ + EventType::EffectsEnabled, + EventType::AggregatorChanged, + EventType::PlaintextAggregated, + EventType::E3StageChanged, + EventType::E3RequestComplete, + EventType::Shutdown, + ], + addr.into(), + ); + } + + fn is_active_aggregator_for(&self, e3_id: &E3id) -> bool { + self.active_aggregators.get(e3_id).copied().unwrap_or(false) + } +} + +impl Actor for InterfoldSolWriter

{ + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } +} diff --git a/crates/evm/src/interfold_writing/effects.rs b/crates/evm/src/interfold_writing/effects.rs new file mode 100644 index 0000000000..f4a8d798be --- /dev/null +++ b/crates/evm/src/interfold_writing/effects.rs @@ -0,0 +1,94 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Interfold contract reads and transaction effects. + +use super::*; + +pub(in crate::actors::interfold_sol_writer) async fn publish_plaintext_output< + P: Provider + WalletProvider + Clone, +>( + provider: EthProvider

, + contract_address: Address, + e3_id: E3id, + decrypted_output: Vec, + decryption_aggregator_proof: Option<&Proof>, +) -> Result { + let e3_id: U256 = e3_id.try_into()?; + + // `None` => proof aggregation disabled; contract accepts empty bytes in that case. + let proof: Bytes = match decryption_aggregator_proof { + Some(p) => encode_zk_proof(p)?, + None => Bytes::new(), + }; + + send_tx_with_retry( + "publishPlaintextOutput", + &["CiphertextOutputNotPublished"], + || { + info!("publishPlaintextOutput() e3_id={:?}", e3_id); + let decrypted_output = Bytes::from(decrypted_output.clone()); + let proof = proof.clone(); + let provider = provider.clone(); + + async move { + let _nonce_guard = transaction_nonce_guard(&provider).await; + let from_address = provider.provider().default_signer_address(); + let current_nonce = provider + .provider() + .get_transaction_count(from_address) + .pending() + .await?; + let contract = IInterfold::new(contract_address, provider.provider()); + let builder = contract + .publishPlaintextOutput(e3_id, decrypted_output, proof) + .nonce(current_nonce); + let pending = builder.send().await?; + drop(_nonce_guard); + let receipt = pending.get_receipt().await?; + require_successful_receipt("publish plaintext output", &receipt)?; + Ok(receipt) + } + }, + ) + .await +} + +pub(in crate::actors::interfold_sol_writer) async fn should_publish_plaintext< + P: Provider + WalletProvider + Clone, +>( + provider: EthProvider

, + contract_address: Address, + e3_id: E3id, +) -> Result { + let e3_id: U256 = e3_id.try_into()?; + let contract = IInterfold::new(contract_address, provider.provider()); + let e3 = contract.getE3(e3_id).call().await?; + Ok(e3.plaintextOutput.is_empty()) +} + +pub(in crate::actors::interfold_sol_writer) async fn process_e3_failure< + P: Provider + WalletProvider + Clone, +>( + provider: EthProvider

, + contract_address: Address, + e3_id: E3id, +) -> Result { + let e3_id: U256 = e3_id.try_into()?; + + info!("processE3Failure() e3_id={:?}", e3_id); + + let _nonce_guard = transaction_nonce_guard(&provider).await; + let from_address = provider.provider().default_signer_address(); + let current_nonce = provider + .provider() + .get_transaction_count(from_address) + .pending() + .await?; + let contract = IInterfold::new(contract_address, provider.provider()); + let builder = contract.processE3Failure(e3_id).nonce(current_nonce); + let pending = builder.send().await?; + drop(_nonce_guard); + let receipt = pending.get_receipt().await?; + require_successful_receipt("process E3 failure", &receipt)?; + Ok(receipt) +} diff --git a/crates/evm/src/actors/interfold_sol_writer.rs b/crates/evm/src/interfold_writing/handlers.rs similarity index 55% rename from crates/evm/src/actors/interfold_sol_writer.rs rename to crates/evm/src/interfold_writing/handlers.rs index 6d62863b72..e58c8df16e 100644 --- a/crates/evm/src/actors/interfold_sol_writer.rs +++ b/crates/evm/src/interfold_writing/handlers.rs @@ -1,96 +1,9 @@ // SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. -use crate::contracts::IInterfold; -use crate::domain::error_decoder::format_evm_error; -use crate::domain::plaintext_publication::validate_plaintext_output; -use crate::helpers::{encode_zk_proof, transaction_nonce_guard, EthProvider}; -use crate::send_tx_with_retry; -use actix::prelude::*; -use alloy::{ - primitives::Address, - providers::{Provider, WalletProvider}, -}; -use alloy::{ - primitives::{Bytes, U256}, - rpc::types::TransactionReceipt, -}; -use anyhow::Result; -use e3_events::BusHandle; -use e3_events::E3RequestComplete; -use e3_events::EventType; -use e3_events::InterfoldEvent; -use e3_events::InterfoldEventData; -use e3_events::Shutdown; -use e3_events::{prelude::*, AggregatorChanged, EffectsEnabled}; -use e3_events::{E3Stage, E3StageChanged}; -use e3_events::{E3id, EType, PlaintextAggregated, Proof}; -use e3_utils::{require_successful_receipt, NotifySync, MAILBOX_LIMIT}; -use std::collections::{HashMap, HashSet}; -use tracing::info; +//! Message routing and actor lifecycle. -/// Consumes events from the event bus and calls EVM methods on the Interfold.sol contract -pub struct InterfoldSolWriter

{ - provider: EthProvider

, - contract_address: Address, - bus: BusHandle, - effects_enabled: bool, - active_aggregators: HashMap, - /// E3s whose `publishPlaintextOutput` submission is currently in flight. - /// Guards against firing a second on-chain tx for the same E3 before the - /// first is observed (H13). The on-chain `should_publish_plaintext` - /// preflight remains the authoritative cross-restart idempotency guard; - /// entries are cleared on failure so a genuine retry can proceed. - submitting: HashSet, -} - -impl InterfoldSolWriter

{ - pub fn new( - bus: &BusHandle, - provider: EthProvider

, - contract_address: Address, - ) -> Result { - Ok(Self { - provider, - contract_address, - bus: bus.clone(), - effects_enabled: false, - active_aggregators: HashMap::new(), - submitting: HashSet::new(), - }) - } - - pub fn attach(bus: &BusHandle, provider: EthProvider

, contract_address: Address) { - let addr = InterfoldSolWriter::new(bus, provider, contract_address) - .expect("failed to create InterfoldSolWriter") - .start(); - bus.subscribe_all( - &[ - EventType::EffectsEnabled, - EventType::AggregatorChanged, - EventType::PlaintextAggregated, - EventType::E3StageChanged, - EventType::E3RequestComplete, - EventType::Shutdown, - ], - addr.into(), - ); - } - - fn is_active_aggregator_for(&self, e3_id: &E3id) -> bool { - self.active_aggregators.get(e3_id).copied().unwrap_or(false) - } -} - -impl Actor for InterfoldSolWriter

{ - type Context = actix::Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT) - } -} +use super::effects::*; +use super::*; impl Handler for InterfoldSolWriter

@@ -301,86 +214,3 @@ impl Handler }) } } - -async fn publish_plaintext_output( - provider: EthProvider

, - contract_address: Address, - e3_id: E3id, - decrypted_output: Vec, - decryption_aggregator_proof: Option<&Proof>, -) -> Result { - let e3_id: U256 = e3_id.try_into()?; - - // `None` => proof aggregation disabled; contract accepts empty bytes in that case. - let proof: Bytes = match decryption_aggregator_proof { - Some(p) => encode_zk_proof(p)?, - None => Bytes::new(), - }; - - send_tx_with_retry( - "publishPlaintextOutput", - &["CiphertextOutputNotPublished"], - || { - info!("publishPlaintextOutput() e3_id={:?}", e3_id); - let decrypted_output = Bytes::from(decrypted_output.clone()); - let proof = proof.clone(); - let provider = provider.clone(); - - async move { - let _nonce_guard = transaction_nonce_guard(&provider).await; - let from_address = provider.provider().default_signer_address(); - let current_nonce = provider - .provider() - .get_transaction_count(from_address) - .pending() - .await?; - let contract = IInterfold::new(contract_address, provider.provider()); - let builder = contract - .publishPlaintextOutput(e3_id, decrypted_output, proof) - .nonce(current_nonce); - let pending = builder.send().await?; - drop(_nonce_guard); - let receipt = pending.get_receipt().await?; - require_successful_receipt("publish plaintext output", &receipt)?; - Ok(receipt) - } - }, - ) - .await -} - -async fn should_publish_plaintext( - provider: EthProvider

, - contract_address: Address, - e3_id: E3id, -) -> Result { - let e3_id: U256 = e3_id.try_into()?; - let contract = IInterfold::new(contract_address, provider.provider()); - let e3 = contract.getE3(e3_id).call().await?; - Ok(e3.plaintextOutput.is_empty()) -} - -async fn process_e3_failure( - provider: EthProvider

, - contract_address: Address, - e3_id: E3id, -) -> Result { - let e3_id: U256 = e3_id.try_into()?; - - info!("processE3Failure() e3_id={:?}", e3_id); - - let _nonce_guard = transaction_nonce_guard(&provider).await; - let from_address = provider.provider().default_signer_address(); - let current_nonce = provider - .provider() - .get_transaction_count(from_address) - .pending() - .await?; - let contract = IInterfold::new(contract_address, provider.provider()); - let builder = contract.processE3Failure(e3_id).nonce(current_nonce); - let pending = builder.send().await?; - drop(_nonce_guard); - let receipt = pending.get_receipt().await?; - require_successful_receipt("process E3 failure", &receipt)?; - Ok(receipt) -} diff --git a/crates/evm/src/domain/plaintext_publication.rs b/crates/evm/src/interfold_writing/workflow.rs similarity index 100% rename from crates/evm/src/domain/plaintext_publication.rs rename to crates/evm/src/interfold_writing/workflow.rs diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs index 416faca905..7984acf5cd 100644 --- a/crates/evm/src/lib.rs +++ b/crates/evm/src/lib.rs @@ -8,11 +8,12 @@ //! //! - [`domain`] holds pure, synchronous, unit-testable services (no actix / //! `BusHandle` / provider types in their cores). -//! - [`actors`] holds the thin actix message-passing shells that wire those -//! services together and perform the EVM/provider I/O. +//! - [`actors`] holds the thin actix message-passing shells. +//! - `adapters` holds concrete EVM/provider I/O. //! - [`messages`] holds the actix message and event types exchanged between them. mod actors; +mod adapters; mod contracts; mod domain; mod messages; diff --git a/crates/evm/src/log_fetching/adapter.rs b/crates/evm/src/log_fetching/adapter.rs new file mode 100644 index 0000000000..8dbbb83a41 --- /dev/null +++ b/crates/evm/src/log_fetching/adapter.rs @@ -0,0 +1,264 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::messages::{EvmEventProcessor, EvmLog, InterfoldEvmEvent}; +use alloy::providers::Provider; +use alloy::rpc::types::{Filter, Log}; +use anyhow::{anyhow, Context as _}; +use async_trait::async_trait; +use e3_events::CorrelationId; +use tracing::{debug, info, warn}; + +const GET_LOGS_CHUNK_SIZE: u64 = 10_000; +const GET_LOGS_MAX_RETRIES: u32 = 3; + +/// Trait abstracting provider methods needed for log fetching. +/// Enables unit testing without a real EVM provider. +#[async_trait] +pub(crate) trait LogProvider: Send + Sync { + async fn fetch_logs(&self, filter: &Filter) -> Result, anyhow::Error>; + async fn fetch_block_number(&self) -> Result; + async fn fetch_block_timestamp(&self, block_number: u64) -> Result; +} + +#[async_trait] +impl LogProvider for P { + async fn fetch_logs(&self, filter: &Filter) -> Result, anyhow::Error> { + self.get_logs(filter).await.map_err(|e| anyhow!("{}", e)) + } + async fn fetch_block_number(&self) -> Result { + self.get_block_number().await.map_err(|e| anyhow!("{}", e)) + } + async fn fetch_block_timestamp(&self, block_number: u64) -> Result { + self.get_block_by_number(block_number.into()) + .await + .map_err(|error| anyhow!("failed to fetch block {block_number}: {error}"))? + .map(|block| block.header.timestamp) + .ok_or_else(|| anyhow!("provider returned no block for height {block_number}")) + } +} + +pub(crate) async fn process_log( + provider: &L, + log: Log, + chain_id: u64, + next: &EvmEventProcessor, + timestamp_tracker: &mut TimestampTracker, +) -> Result { + let timestamp = timestamp_tracker.get(provider, log.block_number).await?; + let evt = InterfoldEvmEvent::Log(EvmLog::new(log, chain_id, timestamp)); + let id = evt.get_id(); + debug!("Sending event({})", id); + next.do_send(evt); + Ok(id) +} + +/// Handle a log delivered by the subscription stream. +/// +/// With a positive confirmation depth the subscription is only a wake-up signal. Publishing the +/// raw notification here would make the historical confirmation gate ineffective, so the periodic +/// canonical backfill owns delivery instead. With zero confirmations this preserves the existing +/// low-latency behavior. +pub(crate) async fn process_live_log( + provider: &L, + log: Log, + chain_id: u64, + next: &EvmEventProcessor, + timestamp_tracker: &mut TimestampTracker, + last_block: &mut u64, + confirmations: u64, +) -> Result, anyhow::Error> { + if confirmations > 0 { + debug!( + chain_id, + block_number = log.block_number, + confirmations, + "Deferring live log to confirmed canonical backfill" + ); + return Ok(None); + } + + let block_number = log.block_number; + let id = process_log(provider, log, chain_id, next, timestamp_tracker).await?; + if let Some(block_number) = block_number { + *last_block = (*last_block).max(block_number); + } + Ok(Some(id)) +} + +/// Fetch logs in chunks from `from_block` to `to_block` with retry logic per chunk. +/// Returns the CorrelationId of the last processed event, if any. +pub(crate) async fn fetch_logs_chunked( + provider: &L, + filter: &Filter, + from_block: u64, + to_block: u64, + chain_id: u64, + next: &EvmEventProcessor, + timestamp_tracker: &mut TimestampTracker, +) -> Result, anyhow::Error> { + if to_block < from_block { + return Ok(None); + } + + let total_blocks = to_block - from_block + 1; + let total_chunks = total_blocks.div_ceil(GET_LOGS_CHUNK_SIZE); + + info!( + chain_id, + from_block, to_block, total_chunks, "Fetching logs in chunks" + ); + + let mut cursor = from_block; + let mut last_id: Option = None; + let mut chunk_idx = 0u64; + + while cursor <= to_block { + let chunk_end = (cursor + GET_LOGS_CHUNK_SIZE - 1).min(to_block); + chunk_idx += 1; + + let chunk_filter = filter.clone().from_block(cursor).to_block(chunk_end); + + let mut success = false; + for attempt in 1..=GET_LOGS_MAX_RETRIES { + match provider.fetch_logs(&chunk_filter).await { + Ok(logs) => { + info!( + chain_id, + chunk = chunk_idx, + total_chunks, + from = cursor, + to = chunk_end, + events = logs.len(), + "Fetched log chunk" + ); + for log in logs { + last_id = Some( + process_log(provider, log, chain_id, next, timestamp_tracker).await?, + ); + } + success = true; + break; + } + Err(e) => { + warn!( + chain_id, chunk = chunk_idx, + from = cursor, to = chunk_end, + attempt, max_retries = GET_LOGS_MAX_RETRIES, + error = %e, "Failed to fetch log chunk, retrying" + ); + if attempt < GET_LOGS_MAX_RETRIES { + tokio::time::sleep(std::time::Duration::from_secs(2u64.pow(attempt))).await; + } + } + } + } + + if !success { + return Err(anyhow!( + "Failed to fetch logs for chain {} blocks {}..={} after {} retries", + chain_id, + cursor, + chunk_end, + GET_LOGS_MAX_RETRIES + )); + } + + cursor = chunk_end + 1; + } + + info!(chain_id, chunks_fetched = chunk_idx, "Log fetch complete"); + Ok(last_id) +} + +/// Fetch any blocks between `last_block` and the chain head to fill gaps. +/// Handles blocks missed during reconnection or due to Geth's eth_subscribe +/// silently ignoring the fromBlock parameter. +pub(crate) async fn backfill_to_head( + provider: &L, + filter: &Filter, + chain_id: u64, + next: &EvmEventProcessor, + timestamp_tracker: &mut TimestampTracker, + last_block: &mut u64, + confirmations: u64, +) -> Result<(), anyhow::Error> { + let raw_head = provider + .fetch_block_number() + .await + .map_err(|e| anyhow!("Failed to get block number for gap backfill: {}", e))?; + // Clamp to the confirmed head so we never ingest logs that a reorg of depth + // `confirmations` could still orphan. `confirmations == 0` is a no-op. + let current_head = crate::domain::reorg::confirmed_head(raw_head, confirmations); + + let gap_start = *last_block + 1; + if gap_start > current_head { + return Ok(()); + } + + info!( + chain_id, + from = gap_start, + to = current_head, + blocks = current_head - gap_start + 1, + "Backfilling missed blocks" + ); + + let mut cursor = gap_start; + while cursor <= current_head { + let chunk_end = (cursor + GET_LOGS_CHUNK_SIZE - 1).min(current_head); + + fetch_logs_chunked( + provider, + filter, + cursor, + chunk_end, + chain_id, + next, + timestamp_tracker, + ) + .await?; + + *last_block = chunk_end; + cursor = chunk_end + 1; + } + + Ok(()) +} + +/// Cache utility to keep track of timestamps +pub(crate) struct TimestampTracker { + current: Option<(u64, u64)>, // (block_number, timestamp) +} + +impl TimestampTracker { + pub fn new() -> Self { + Self { current: None } + } + + pub async fn get( + &mut self, + provider: &L, + block_number: Option, + ) -> Result { + let bn = block_number.context("provider log is missing its block number")?; + + if let Some((cached_bn, ts)) = self.current { + if bn == cached_bn { + return Ok(ts); + } + } + + let ts = provider.fetch_block_timestamp(bn).await?; + + self.current = Some((bn, ts)); + Ok(ts) + } +} + +#[cfg(test)] +#[path = "tests/mod.rs"] +mod tests; diff --git a/crates/evm/src/log_fetching/tests/backfill.rs b/crates/evm/src/log_fetching/tests/backfill.rs new file mode 100644 index 0000000000..a555a3b4c1 --- /dev/null +++ b/crates/evm/src/log_fetching/tests/backfill.rs @@ -0,0 +1,145 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn test_backfill_no_gap() { + let mock = MockLogProvider::new(100); + let (next, _rx) = setup_collector(); + let mut ts = TimestampTracker::new(); + let filter = Filter::new(); + let mut last_block = 100u64; + + let result = backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 0).await; + + assert!(result.is_ok()); + assert_eq!(last_block, 100); + assert_eq!(mock.get_logs_call_count(), 0); +} + +#[actix::test] +async fn test_backfill_with_gap() { + let mock = MockLogProvider::new(200); + mock.push_logs(vec![make_test_log(150), make_test_log(180)]); + let (next, mut rx) = setup_collector(); + let mut ts = TimestampTracker::new(); + let filter = Filter::new(); + let mut last_block = 100u64; + + let result = backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 0).await; + + assert!(result.is_ok()); + assert_eq!(last_block, 200); + assert_eq!(mock.get_logs_call_count(), 1); + + tokio::task::yield_now().await; + let mut count = 0; + while rx.try_recv().is_ok() { + count += 1; + } + assert_eq!(count, 2); +} + +#[actix::test] +async fn test_backfill_partial_failure_preserves_progress() { + tokio::time::pause(); + + // Head at 25000, last_block at 100 + // Gap: blocks 101..=25000 → 3 chunks: + // chunk 1: [101, 10100] + // chunk 2: [10101, 20100] + // chunk 3: [20101, 25000] + let mock = MockLogProvider::new(25000); + // Chunk 1 succeeds + mock.push_logs(vec![make_test_log(500)]); + // Chunk 2 succeeds + mock.push_logs(vec![make_test_log(15000)]); + // Chunk 3: all retries fail + for _ in 0..GET_LOGS_MAX_RETRIES { + mock.push_error("RPC error"); + } + + let (next, _rx) = setup_collector(); + let mut ts = TimestampTracker::new(); + let filter = Filter::new(); + let mut last_block = 100u64; + + let result = backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 0).await; + + // Should fail because chunk 3 exhausted retries + assert!(result.is_err()); + // But last_block must have advanced past the two successful chunks + assert_eq!(last_block, 20100); + + // On retry: gap_start = 20101, head still 25000 → single chunk succeeds + mock.push_logs(vec![make_test_log(22000)]); + + let result = backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 0).await; + assert!(result.is_ok()); + assert_eq!(last_block, 25000); +} + +#[actix::test] +async fn test_backfill_clamps_to_confirmed_head() { + // Head at 200, but require 12 confirmations => only ingest up to 188. + let mock = MockLogProvider::new(200); + mock.push_logs(vec![make_test_log(150)]); + let (next, _rx) = setup_collector(); + let mut ts = TimestampTracker::new(); + let filter = Filter::new(); + let mut last_block = 100u64; + + let result = backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 12).await; + + assert!(result.is_ok()); + // Advanced only to the confirmed head, not the raw head of 200. + assert_eq!(last_block, 188); +} + +#[actix::test] +async fn live_log_waits_for_confirmed_canonical_backfill() -> anyhow::Result<()> { + let mock = MockLogProvider::new(200); + let (next, mut rx) = setup_collector(); + let mut ts = TimestampTracker::new(); + let filter = Filter::new(); + let mut last_block = 188; + + let delivered = process_live_log( + &mock, + make_test_log(200), + 1, + &next, + &mut ts, + &mut last_block, + 12, + ) + .await?; + assert!(delivered.is_none()); + assert_eq!(last_block, 188); + tokio::task::yield_now().await; + assert!( + rx.try_recv().is_err(), + "unconfirmed log must not be emitted" + ); + + mock.set_block_number(211); + mock.push_logs(Vec::new()); + backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 12).await?; + assert_eq!(last_block, 199); + tokio::task::yield_now().await; + assert!(rx.try_recv().is_err(), "eleven blocks is not enough"); + + mock.set_block_number(212); + mock.push_logs(vec![make_test_log(200)]); + backfill_to_head(&mock, &filter, 1, &next, &mut ts, &mut last_block, 12).await?; + assert_eq!(last_block, 200); + let emitted = tokio::time::timeout(std::time::Duration::from_secs(1), rx.recv()) + .await? + .expect("confirmed log should be emitted"); + assert!(matches!(emitted, InterfoldEvmEvent::Log(_))); + Ok(()) +} diff --git a/crates/evm/src/log_fetching/tests/fetch.rs b/crates/evm/src/log_fetching/tests/fetch.rs new file mode 100644 index 0000000000..86ae7501ab --- /dev/null +++ b/crates/evm/src/log_fetching/tests/fetch.rs @@ -0,0 +1,147 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn test_fetch_logs_empty_range() { + let mock = MockLogProvider::new(100); + let (next, _rx) = setup_collector(); + let mut ts = TimestampTracker::new(); + let filter = Filter::new(); + + let result = fetch_logs_chunked(&mock, &filter, 200, 100, 1, &next, &mut ts).await; + + assert!(result.is_ok()); + assert!(result.unwrap().is_none()); + assert_eq!(mock.get_logs_call_count(), 0); +} + +#[actix::test] +async fn timestamp_failure_is_not_cached_as_zero() { + let mock = MockLogProvider::new(100); + mock.push_timestamp_error("RPC unavailable"); + mock.push_timestamp(1234); + let mut tracker = TimestampTracker::new(); + + let error = tracker.get(&mock, Some(100)).await.unwrap_err(); + assert!(error.to_string().contains("RPC unavailable")); + assert_eq!(tracker.get(&mock, Some(100)).await.unwrap(), 1234); + assert_eq!(tracker.get(&mock, Some(100)).await.unwrap(), 1234); + assert_eq!(mock.timestamp_call_count(), 2); +} + +#[actix::test] +async fn missing_log_block_number_is_rejected_without_rpc_fallback() { + let mock = MockLogProvider::new(100); + let mut tracker = TimestampTracker::new(); + + let error = tracker.get(&mock, None).await.unwrap_err(); + assert!(error.to_string().contains("missing its block number")); + assert_eq!(mock.timestamp_call_count(), 0); +} + +#[actix::test] +async fn timestamp_rpc_failure_prevents_log_dispatch() { + let mock = MockLogProvider::new(100); + mock.push_logs(vec![make_test_log(100)]); + mock.push_timestamp_error("timestamp RPC unavailable"); + let (next, mut receiver) = setup_collector(); + let mut tracker = TimestampTracker::new(); + + let error = fetch_logs_chunked(&mock, &Filter::new(), 100, 100, 1, &next, &mut tracker) + .await + .unwrap_err(); + + assert!(error.to_string().contains("timestamp RPC unavailable")); + tokio::task::yield_now().await; + assert!(receiver.try_recv().is_err()); +} + +#[actix::test] +async fn test_fetch_logs_single_chunk() { + let mock = MockLogProvider::new(5000); + mock.push_logs(vec![ + make_test_log(100), + make_test_log(200), + make_test_log(300), + ]); + let (next, mut rx) = setup_collector(); + let mut ts = TimestampTracker::new(); + let filter = Filter::new(); + + let result = fetch_logs_chunked(&mock, &filter, 0, 5000, 1, &next, &mut ts).await; + + assert!(result.is_ok()); + assert!(result.unwrap().is_some()); + assert_eq!(mock.get_logs_call_count(), 1); + + // Allow actix message delivery + tokio::task::yield_now().await; + let mut count = 0; + while rx.try_recv().is_ok() { + count += 1; + } + assert_eq!(count, 3); +} + +#[actix::test] +async fn test_fetch_logs_multiple_chunks() { + // 25k blocks → 3 chunks: [0..9999], [10000..19999], [20000..24999] + let mock = MockLogProvider::new(25000); + mock.push_logs(vec![make_test_log(5000)]); + mock.push_logs(vec![make_test_log(15000)]); + mock.push_logs(vec![make_test_log(22000)]); + let (next, _rx) = setup_collector(); + let mut ts = TimestampTracker::new(); + let filter = Filter::new(); + + let result = fetch_logs_chunked(&mock, &filter, 0, 24999, 1, &next, &mut ts).await; + + assert!(result.is_ok()); + assert!(result.unwrap().is_some()); + assert_eq!(mock.get_logs_call_count(), 3); +} + +#[actix::test] +async fn test_fetch_logs_retry_then_success() { + tokio::time::pause(); // Skip retry delays + + let mock = MockLogProvider::new(5000); + mock.push_error("temporary RPC error"); + mock.push_logs(vec![make_test_log(100)]); + let (next, _rx) = setup_collector(); + let mut ts = TimestampTracker::new(); + let filter = Filter::new(); + + let result = fetch_logs_chunked(&mock, &filter, 0, 5000, 1, &next, &mut ts).await; + + assert!(result.is_ok()); + assert!(result.unwrap().is_some()); + assert_eq!(mock.get_logs_call_count(), 2); +} + +#[actix::test] +async fn test_fetch_logs_all_retries_exhausted() { + tokio::time::pause(); + + let mock = MockLogProvider::new(5000); + for _ in 0..GET_LOGS_MAX_RETRIES { + mock.push_error("persistent RPC error"); + } + let (next, _rx) = setup_collector(); + let mut ts = TimestampTracker::new(); + let filter = Filter::new(); + + let result = fetch_logs_chunked(&mock, &filter, 0, 5000, 1, &next, &mut ts).await; + + let err = result.expect_err("expected error after all retries exhausted"); + assert!( + err.to_string().contains("Failed to fetch logs"), + "unexpected error: {err}" + ); + assert_eq!(mock.get_logs_call_count(), GET_LOGS_MAX_RETRIES); +} diff --git a/crates/evm/src/log_fetching/tests/mod.rs b/crates/evm/src/log_fetching/tests/mod.rs new file mode 100644 index 0000000000..e1222bfa01 --- /dev/null +++ b/crates/evm/src/log_fetching/tests/mod.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use actix::prelude::*; +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; +use tokio::sync::mpsc; + +#[derive(Clone)] +struct MockLogProvider { + inner: Arc>, +} + +struct MockState { + block_number: u64, + log_responses: VecDeque, String>>, + timestamp_responses: VecDeque>, + timestamp_calls: u32, + get_logs_calls: u32, +} + +impl MockLogProvider { + fn new(block_number: u64) -> Self { + Self { + inner: Arc::new(Mutex::new(MockState { + block_number, + log_responses: VecDeque::new(), + timestamp_responses: VecDeque::new(), + timestamp_calls: 0, + get_logs_calls: 0, + })), + } + } + + fn push_logs(&self, logs: Vec) { + self.inner.lock().unwrap().log_responses.push_back(Ok(logs)); + } + + fn push_error(&self, msg: &str) { + self.inner + .lock() + .unwrap() + .log_responses + .push_back(Err(msg.to_string())); + } + + fn get_logs_call_count(&self) -> u32 { + self.inner.lock().unwrap().get_logs_calls + } + + fn push_timestamp(&self, timestamp: u64) { + self.inner + .lock() + .unwrap() + .timestamp_responses + .push_back(Ok(timestamp)); + } + + fn push_timestamp_error(&self, message: &str) { + self.inner + .lock() + .unwrap() + .timestamp_responses + .push_back(Err(message.to_string())); + } + + fn timestamp_call_count(&self) -> u32 { + self.inner.lock().unwrap().timestamp_calls + } + + fn set_block_number(&self, block_number: u64) { + self.inner.lock().unwrap().block_number = block_number; + } +} + +#[async_trait] +impl LogProvider for MockLogProvider { + async fn fetch_logs(&self, _filter: &Filter) -> Result, anyhow::Error> { + let mut state = self.inner.lock().unwrap(); + state.get_logs_calls += 1; + match state.log_responses.pop_front() { + Some(Ok(logs)) => Ok(logs), + Some(Err(msg)) => Err(anyhow!("{}", msg)), + None => Ok(vec![]), + } + } + + async fn fetch_block_number(&self) -> Result { + Ok(self.inner.lock().unwrap().block_number) + } + + async fn fetch_block_timestamp(&self, _block_number: u64) -> Result { + let mut state = self.inner.lock().unwrap(); + state.timestamp_calls += 1; + match state.timestamp_responses.pop_front() { + Some(Ok(timestamp)) => Ok(timestamp), + Some(Err(message)) => Err(anyhow!(message)), + None => Ok(0), + } + } +} + +struct TestCollector { + tx: mpsc::UnboundedSender, +} + +impl Actor for TestCollector { + type Context = Context; +} + +impl Handler for TestCollector { + type Result = (); + fn handle(&mut self, msg: InterfoldEvmEvent, _: &mut Self::Context) { + let _ = self.tx.send(msg); + } +} + +fn make_test_log(block_number: u64) -> Log { + Log { + block_number: Some(block_number), + ..Default::default() + } +} + +fn setup_collector() -> ( + EvmEventProcessor, + mpsc::UnboundedReceiver, +) { + let (tx, rx) = mpsc::unbounded_channel(); + let addr = TestCollector { tx }.start(); + (addr.recipient(), rx) +} + +mod backfill; +mod fetch; diff --git a/crates/evm/src/domain/slashing_events.rs b/crates/evm/src/slashing/events.rs similarity index 100% rename from crates/evm/src/domain/slashing_events.rs rename to crates/evm/src/slashing/events.rs diff --git a/crates/evm/src/domain/attestation_evidence.rs b/crates/evm/src/slashing/evidence.rs similarity index 100% rename from crates/evm/src/domain/attestation_evidence.rs rename to crates/evm/src/slashing/evidence.rs diff --git a/crates/evm/src/actors/slashing_manager_sol_reader.rs b/crates/evm/src/slashing/reader.rs similarity index 100% rename from crates/evm/src/actors/slashing_manager_sol_reader.rs rename to crates/evm/src/slashing/reader.rs diff --git a/crates/evm/src/slashing_writing/actor.rs b/crates/evm/src/slashing_writing/actor.rs new file mode 100644 index 0000000000..e97fcaa2f4 --- /dev/null +++ b/crates/evm/src/slashing_writing/actor.rs @@ -0,0 +1,104 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Subscribes to `AccusationQuorumReached` events and submits committee-attested +//! slash proposals on the SlashingManager contract. Prefers party-attributed +//! `proposeSlashByDkgParty` when DKG anchors resolve, and falls back to +//! operator-attributed `proposeSlash` otherwise. + +use crate::contracts::{ICiphernodeRegistry, ISlashingManager}; +use crate::domain::attestation_evidence::encode_attestation_evidence; +use crate::domain::error_decoder::format_evm_error; +use crate::domain::slash_submission::{ + should_submit_slash, submission_delay, submission_rank, SlashIntentKey, + SlashSubmissionDecision, SlashSubmissionGate, +}; +use crate::helpers::{transaction_nonce_guard, EthProvider}; +use crate::send_tx_with_retry; +use actix::prelude::*; +use actix::Addr; +use alloy::{ + primitives::{Address, Bytes, U256}, + providers::{Provider, WalletProvider}, + rpc::types::TransactionReceipt, +}; +use anyhow::Result; +use e3_events::prelude::*; +use e3_events::BusHandle; +use e3_events::EventType; +use e3_events::InterfoldEvent; +use e3_events::InterfoldEventData; +use e3_events::Shutdown; +use e3_events::{AccusationQuorumReached, EType}; +use e3_utils::{require_successful_receipt, NotifySync, MAILBOX_LIMIT}; +use tracing::{info, warn}; + +#[path = "effects.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +/// Submits `AccusationQuorumReached` events as slash proposals on-chain. +pub struct SlashingManagerSolWriter

{ + provider: EthProvider

, + contract_address: Address, + bus: BusHandle, + submissions: SlashSubmissionGate, +} + +#[derive(Message)] +#[rtype(result = "()")] +struct SubmitSlashIntent { + key: SlashIntentKey, + event: AccusationQuorumReached, +} + +#[derive(Message)] +#[rtype(result = "()")] +struct SlashSubmissionFinished { + key: SlashIntentKey, + terminal: bool, +} + +impl SlashingManagerSolWriter

{ + pub fn new( + bus: &BusHandle, + provider: EthProvider

, + contract_address: Address, + ) -> Result { + Ok(Self { + provider, + contract_address, + bus: bus.clone(), + submissions: SlashSubmissionGate::new(), + }) + } + + pub async fn attach( + bus: &BusHandle, + provider: EthProvider

, + contract_address: Address, + ) -> Result>> { + let addr = SlashingManagerSolWriter::new(bus, provider, contract_address)?.start(); + bus.subscribe_all( + &[ + EventType::AccusationQuorumReached, + EventType::EffectsEnabled, + EventType::Shutdown, + ], + addr.clone().into(), + ); + Ok(addr) + } +} + +impl Actor for SlashingManagerSolWriter

{ + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } +} diff --git a/crates/evm/src/slashing_writing/effects.rs b/crates/evm/src/slashing_writing/effects.rs new file mode 100644 index 0000000000..cc700095ea --- /dev/null +++ b/crates/evm/src/slashing_writing/effects.rs @@ -0,0 +1,111 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! SlashingManager contract effects. + +use super::*; + +pub(in crate::actors::slashing_manager_sol_writer) async fn submit_slash_proposal< + P: Provider + WalletProvider + Clone, +>( + provider: EthProvider

, + contract_address: Address, + data: AccusationQuorumReached, +) -> Result { + let e3_id: U256 = data.e3_id.clone().try_into()?; + let operator = data.accused; + + // Empty `votes_for` only reaches this point if upstream invariants broke + // — `check_quorum` requires `len >= threshold_m >= 1` before emitting + // `AccusedFaulted`/`Equivocation`. Refuse to submit malformed calldata + // and surface a structured warning so an operator can debug the upstream + // gossip/quorum path rather than seeing a generic ABI-decode revert + // on chain. + let proof_data = match encode_attestation_evidence(&data) { + Some(bytes) => bytes, + None => { + warn!( + e3_id = %data.e3_id, + accused = %operator, + outcome = %data.outcome, + "Refusing to submit proposeSlash: AccusationQuorumReached has empty \ + votes_for or empty evidence preimage — submission dropped" + ); + return Err(anyhow::anyhow!( + "AccusationQuorumReached has empty votes_for or evidence; refused proposeSlash submission \ + (e3_id={}, accused={})", + data.e3_id, + operator + )); + } + }; + + let party_id = + resolve_party_id_for_operator(provider.clone(), contract_address, e3_id, operator) + .await + .ok() + .flatten(); + + send_tx_with_retry("proposeSlash", &[], || { + info!( + "proposeSlash() e3_id={:?} operator={:?} party_id={:?}", + e3_id, operator, party_id + ); + let proof = Bytes::from(proof_data.clone()); + let provider = provider.clone(); + + async move { + let _nonce_guard = transaction_nonce_guard(&provider).await; + let from_address = provider.provider().default_signer_address(); + let current_nonce = provider + .provider() + .get_transaction_count(from_address) + .pending() + .await?; + let contract = ISlashingManager::new(contract_address, provider.provider()); + let pending = if let Some(pid) = party_id { + contract + .proposeSlashByDkgParty(e3_id, pid, proof) + .nonce(current_nonce) + .send() + .await? + } else { + contract + .proposeSlash(e3_id, operator, proof) + .nonce(current_nonce) + .send() + .await? + }; + drop(_nonce_guard); + let receipt = pending.get_receipt().await?; + require_successful_receipt("submit slashing evidence", &receipt)?; + Ok(receipt) + } + }) + .await +} + +async fn resolve_party_id_for_operator( + provider: EthProvider

, + contract_address: Address, + e3_id: U256, + operator: Address, +) -> Result> { + let slashing = ISlashingManager::new(contract_address, provider.provider()); + let registry = slashing.ciphernodeRegistry().call().await?; + if registry == Address::ZERO { + return Ok(None); + } + + let registry_view = ICiphernodeRegistry::new(registry, provider.provider()); + let anchors = registry_view.getDkgAnchors(e3_id).call().await?; + for pid in anchors.partyIds { + let node = registry_view + .canonicalCommitteeNodeAt(e3_id, pid) + .call() + .await?; + if node == operator { + return Ok(Some(pid)); + } + } + Ok(None) +} diff --git a/crates/evm/src/actors/slashing_manager_sol_writer.rs b/crates/evm/src/slashing_writing/handlers.rs similarity index 51% rename from crates/evm/src/actors/slashing_manager_sol_writer.rs rename to crates/evm/src/slashing_writing/handlers.rs index 94da6954f1..09a86364b6 100644 --- a/crates/evm/src/actors/slashing_manager_sol_writer.rs +++ b/crates/evm/src/slashing_writing/handlers.rs @@ -1,102 +1,9 @@ // SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. -//! Subscribes to `AccusationQuorumReached` events and submits committee-attested -//! slash proposals on the SlashingManager contract. Prefers party-attributed -//! `proposeSlashByDkgParty` when DKG anchors resolve, and falls back to -//! operator-attributed `proposeSlash` otherwise. +//! Admission, scheduling, and submission-outcome handlers. -use crate::contracts::{ICiphernodeRegistry, ISlashingManager}; -use crate::domain::attestation_evidence::encode_attestation_evidence; -use crate::domain::error_decoder::format_evm_error; -use crate::domain::slash_submission::{ - should_submit_slash, submission_delay, submission_rank, SlashIntentKey, - SlashSubmissionDecision, SlashSubmissionGate, -}; -use crate::helpers::{transaction_nonce_guard, EthProvider}; -use crate::send_tx_with_retry; -use actix::prelude::*; -use actix::Addr; -use alloy::{ - primitives::{Address, Bytes, U256}, - providers::{Provider, WalletProvider}, - rpc::types::TransactionReceipt, -}; -use anyhow::Result; -use e3_events::prelude::*; -use e3_events::BusHandle; -use e3_events::EventType; -use e3_events::InterfoldEvent; -use e3_events::InterfoldEventData; -use e3_events::Shutdown; -use e3_events::{AccusationQuorumReached, EType}; -use e3_utils::{require_successful_receipt, NotifySync, MAILBOX_LIMIT}; -use tracing::{info, warn}; - -/// Submits `AccusationQuorumReached` events as slash proposals on-chain. -pub struct SlashingManagerSolWriter

{ - provider: EthProvider

, - contract_address: Address, - bus: BusHandle, - submissions: SlashSubmissionGate, -} - -#[derive(Message)] -#[rtype(result = "()")] -struct SubmitSlashIntent { - key: SlashIntentKey, - event: AccusationQuorumReached, -} - -#[derive(Message)] -#[rtype(result = "()")] -struct SlashSubmissionFinished { - key: SlashIntentKey, - terminal: bool, -} - -impl SlashingManagerSolWriter

{ - pub fn new( - bus: &BusHandle, - provider: EthProvider

, - contract_address: Address, - ) -> Result { - Ok(Self { - provider, - contract_address, - bus: bus.clone(), - submissions: SlashSubmissionGate::new(), - }) - } - - pub async fn attach( - bus: &BusHandle, - provider: EthProvider

, - contract_address: Address, - ) -> Result>> { - let addr = SlashingManagerSolWriter::new(bus, provider, contract_address)?.start(); - bus.subscribe_all( - &[ - EventType::AccusationQuorumReached, - EventType::EffectsEnabled, - EventType::Shutdown, - ], - addr.clone().into(), - ); - Ok(addr) - } -} - -impl Actor for SlashingManagerSolWriter

{ - type Context = actix::Context; - - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT); - } -} +use super::effects::submit_slash_proposal; +use super::*; impl Handler for SlashingManagerSolWriter

@@ -260,107 +167,3 @@ impl Handler ctx.stop(); } } - -async fn submit_slash_proposal( - provider: EthProvider

, - contract_address: Address, - data: AccusationQuorumReached, -) -> Result { - let e3_id: U256 = data.e3_id.clone().try_into()?; - let operator = data.accused; - - // Empty `votes_for` only reaches this point if upstream invariants broke - // — `check_quorum` requires `len >= threshold_m >= 1` before emitting - // `AccusedFaulted`/`Equivocation`. Refuse to submit malformed calldata - // and surface a structured warning so an operator can debug the upstream - // gossip/quorum path rather than seeing a generic ABI-decode revert - // on chain. - let proof_data = match encode_attestation_evidence(&data) { - Some(bytes) => bytes, - None => { - warn!( - e3_id = %data.e3_id, - accused = %operator, - outcome = %data.outcome, - "Refusing to submit proposeSlash: AccusationQuorumReached has empty \ - votes_for or empty evidence preimage — submission dropped" - ); - return Err(anyhow::anyhow!( - "AccusationQuorumReached has empty votes_for or evidence; refused proposeSlash submission \ - (e3_id={}, accused={})", - data.e3_id, - operator - )); - } - }; - - let party_id = - resolve_party_id_for_operator(provider.clone(), contract_address, e3_id, operator) - .await - .ok() - .flatten(); - - send_tx_with_retry("proposeSlash", &[], || { - info!( - "proposeSlash() e3_id={:?} operator={:?} party_id={:?}", - e3_id, operator, party_id - ); - let proof = Bytes::from(proof_data.clone()); - let provider = provider.clone(); - - async move { - let _nonce_guard = transaction_nonce_guard(&provider).await; - let from_address = provider.provider().default_signer_address(); - let current_nonce = provider - .provider() - .get_transaction_count(from_address) - .pending() - .await?; - let contract = ISlashingManager::new(contract_address, provider.provider()); - let pending = if let Some(pid) = party_id { - contract - .proposeSlashByDkgParty(e3_id, pid, proof) - .nonce(current_nonce) - .send() - .await? - } else { - contract - .proposeSlash(e3_id, operator, proof) - .nonce(current_nonce) - .send() - .await? - }; - drop(_nonce_guard); - let receipt = pending.get_receipt().await?; - require_successful_receipt("submit slashing evidence", &receipt)?; - Ok(receipt) - } - }) - .await -} - -async fn resolve_party_id_for_operator( - provider: EthProvider

, - contract_address: Address, - e3_id: U256, - operator: Address, -) -> Result> { - let slashing = ISlashingManager::new(contract_address, provider.provider()); - let registry = slashing.ciphernodeRegistry().call().await?; - if registry == Address::ZERO { - return Ok(None); - } - - let registry_view = ICiphernodeRegistry::new(registry, provider.provider()); - let anchors = registry_view.getDkgAnchors(e3_id).call().await?; - for pid in anchors.partyIds { - let node = registry_view - .canonicalCommitteeNodeAt(e3_id, pid) - .call() - .await?; - if node == operator { - return Ok(Some(pid)); - } - } - Ok(None) -} diff --git a/crates/evm/src/domain/slash_submission.rs b/crates/evm/src/slashing_writing/workflow.rs similarity index 100% rename from crates/evm/src/domain/slash_submission.rs rename to crates/evm/src/slashing_writing/workflow.rs diff --git a/crates/keyshare/src/actors/mod.rs b/crates/keyshare/src/actors.rs similarity index 65% rename from crates/keyshare/src/actors/mod.rs rename to crates/keyshare/src/actors.rs index b9e8c9f3c6..bfd6074dd2 100644 --- a/crates/keyshare/src/actors/mod.rs +++ b/crates/keyshare/src/actors.rs @@ -4,14 +4,18 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -//! Actix actor shells for the threshold keyshare flow. +//! Compatibility view of the actors stored in `threshold_keyshare`. //! //! These actors own mailboxes, timers, persistence and bus I/O. All -//! business/decision logic lives in [`crate::domain`]. +//! business/decision logic lives beside them in the capability directory. +#[path = "threshold_keyshare/decryption_key_collector.rs"] pub(crate) mod decryption_key_shared_collector; +#[path = "threshold_keyshare/encryption_key_collector.rs"] pub(crate) mod encryption_key_collector; +#[path = "threshold_keyshare/actor.rs"] pub(crate) mod threshold_keyshare; +#[path = "threshold_keyshare/threshold_share_collector.rs"] pub(crate) mod threshold_share_collector; pub use encryption_key_collector::{ diff --git a/crates/keyshare/src/actors/threshold_keyshare.rs b/crates/keyshare/src/actors/threshold_keyshare.rs deleted file mode 100644 index cbb04abf96..0000000000 --- a/crates/keyshare/src/actors/threshold_keyshare.rs +++ /dev/null @@ -1,2372 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use actix::prelude::*; -use anyhow::{anyhow, bail, Context, Result}; -use e3_crypto::{Cipher, SensitiveBytes}; -use e3_data::Persistable; -use e3_events::{ - prelude::*, trap, BusHandle, CiphernodeSelected, CiphertextOutputPublished, - CommitteeMemberExpelled, ComputeRequest, ComputeResponse, ComputeResponseKind, CorrelationId, - DecryptionKeyShared, DecryptionShareProofSigned, DecryptionShareProofsPending, Die, - DkgProofSigned, DkgShareDecryptionProofRequest, E3Failed, E3RequestComplete, E3Stage, EType, - EncryptionKey, EncryptionKeyCollectionFailed, EncryptionKeyCreated, EncryptionKeyPending, - EventContext, FailureReason, InterfoldEvent, InterfoldEventData, KeyshareCreated, - PartyProofsToVerify, PartyShareDecryptionProofsToVerify, PkGenerationProofSigned, ProofType, - Sequenced, ShareDecryptionProofPending, ShareVerificationComplete, ShareVerificationDispatched, - SignedProofPayload, ThresholdShare, ThresholdShareCollectionFailed, ThresholdShareCreated, - ThresholdShareDecryptionProofRequest, ThresholdSharePending, TypedEvent, VerificationKind, -}; -use e3_fhe_params::create_deterministic_crp_from_default_seed; -use e3_fhe_params::BfvPreset; -use e3_trbfv::{ - calculate_decryption_key::CalculateDecryptionKeyResponse, - calculate_decryption_share::{ - CalculateDecryptionShareRequest, CalculateDecryptionShareResponse, - }, - gen_esi_sss::{GenEsiSssRequest, GenEsiSssResponse}, - gen_pk_share_and_sk_sss::{GenPkShareAndSkSssRequest, GenPkShareAndSkSssResponse}, - shares::SharedSecret, - TrBFVConfig, TrBFVRequest, TrBFVResponse, -}; -use e3_utils::utility_types::ArcBytes; -use e3_utils::{NotifySync, MAILBOX_LIMIT}; -use e3_zk_helpers::CiphernodesCommitteeSize; -use fhe_traits::Serialize; -use std::{ - collections::{BTreeSet, HashMap, HashSet}, - sync::Arc, -}; -use tracing::{error, info, trace, warn}; - -use crate::actors::decryption_key_shared_collector::{ - AllDecryptionKeySharesCollected, DecryptionKeySharedCollectionFailed, - DecryptionKeySharedCollector, ExpelPartyFromDecryptionKeySharedCollection, -}; -use crate::actors::encryption_key_collector::{ - AllEncryptionKeysCollected, EncryptionKeyCollector, ExpelPartyFromKeyCollection, -}; -use crate::actors::threshold_share_collector::{ - ExpelPartyFromShareCollection, ThresholdShareCollector, -}; -use crate::domain::timeout_policy::{resolve_timeout, DkgTimeoutPhase}; -use crate::domain::{ - build_decryption_key_plan, build_shares_generated_plan, generate_bfv_keypair, - AggregatingDecryptionKey, BfvKeypairMaterial, CollectingEncryptionKeysData, Decrypting, - DecryptionKeyPlan, GeneratingDecryptionProof, GeneratingThresholdShareData, KeyshareState, - ProofRequestData, ReadyForDecryption, ReceivedShareProofs, ThresholdKeyshareState, -}; - -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[rtype(result = "()")] -pub struct GenPkShareAndSkSss(CiphernodeSelected); - -#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] -#[rtype(result = "()")] -pub struct GenEsiSss { - pub ciphernode_selected: CiphernodeSelected, - pub e_sm_raw: SensitiveBytes, -} - -#[derive(Message)] -#[rtype(result = "()")] -pub struct AllThresholdSharesCollected { - /// Threshold shares sorted by ascending `party_id`. - shares: Vec>, - /// Proofs from each sender, ordered by party_id (parallel to shares). - share_proofs: Vec, -} - -impl AllThresholdSharesCollected { - pub fn new( - shares: HashMap>, - proofs: HashMap, - ) -> Self { - let mut entries: Vec<_> = shares.into_iter().collect(); - entries.sort_by_key(|(k, _)| *k); - let (party_ids, shares): (Vec<_>, Vec<_>) = entries.into_iter().unzip(); - let share_proofs = party_ids - .iter() - .map(|pid| { - proofs.get(pid).cloned().unwrap_or(ReceivedShareProofs { - signed_c2a_proof: None, - signed_c2b_proof: None, - signed_c3a_proofs: Vec::new(), - signed_c3b_proofs: Vec::new(), - }) - }) - .collect(); - Self { - shares, - share_proofs, - } - } -} - -pub struct ThresholdKeyshareParams { - pub bus: BusHandle, - pub cipher: Arc, - pub state: Persistable, - pub share_enc_preset: BfvPreset, -} - -pub struct ThresholdKeyshare { - bus: BusHandle, - cipher: Arc, - decryption_key_collector: Option>, - encryption_key_collector: Option>, - decryption_key_shared_collector: Option>, - state: Persistable, - share_enc_preset: BfvPreset, - /// Transient coordination data bridging async gaps — not persisted. - /// Shares pending C2/C3 verification, consumed in `proceed_with_decryption_key_calculation`. - pending_shares: Vec>, - /// Share decryption proof data built during aggregation, consumed after CalculateDecryptionKey. - pending_share_decryption_data: Option<( - DkgShareDecryptionProofRequest, - Vec, - )>, - /// Temporarily stores DecryptionKeyShared while C4 verification is in flight. - pending_c4_verification_shares: Option>, - /// Own DKG plaintext shares captured during `handle_shares_generated`, consumed by - /// the AggregatingDecryptionKey transition. Tuple is `(own_sk_share, own_esi_shares)`. - /// Each entry is bincode-encoded `Vec>` of shape `[L][N]`. - pending_own_dkg_shares: Option<(SensitiveBytes, Vec)>, - /// Set when C4 verification completes before `PkGenerationProofSigned` is applied. - pending_keyshare_publish: bool, -} - -impl ThresholdKeyshare { - pub fn new(params: ThresholdKeyshareParams) -> Self { - Self { - bus: params.bus, - cipher: params.cipher, - decryption_key_collector: None, - encryption_key_collector: None, - decryption_key_shared_collector: None, - state: params.state, - share_enc_preset: params.share_enc_preset, - pending_shares: Vec::new(), - pending_share_decryption_data: None, - pending_c4_verification_shares: None, - pending_own_dkg_shares: None, - pending_keyshare_publish: false, - } - } - - fn store_signed_pk_generation_proof( - &mut self, - ec: &EventContext, - signed: SignedProofPayload, - ) -> Result<()> { - self.state.try_mutate(ec, |mut s| { - match &mut s.state { - KeyshareState::AggregatingDecryptionKey(adk) => { - adk.signed_pk_generation_proof = Some(signed.clone()); - } - KeyshareState::ReadyForDecryption(rfd) => { - rfd.signed_pk_generation_proof = Some(signed.clone()); - } - KeyshareState::Decrypting(d) => { - d.signed_pk_generation_proof = Some(signed.clone()); - } - other => { - warn!( - "PkGenerationProofSigned in {:?} — C1 proof not stored (unexpected state)", - other.variant_name() - ); - } - } - Ok(s) - }) - } - - fn keyshare_created_fields( - state: &KeyshareState, - ) -> Option<(&ArcBytes, &Option)> { - use KeyshareState as K; - match state { - K::ReadyForDecryption(s) => Some((&s.pk_share, &s.signed_pk_generation_proof)), - K::Decrypting(s) => Some((&s.pk_share, &s.signed_pk_generation_proof)), - _ => None, - } - } - - fn try_finish_deferred_keyshare_publish(&mut self, ec: EventContext) -> Result<()> { - if !self.pending_keyshare_publish { - return Ok(()); - } - let state = self.state.try_get()?; - let Some((_, signed)) = Self::keyshare_created_fields(&state.state) else { - return Ok(()); - }; - if signed.is_none() { - return Ok(()); - } - self.pending_keyshare_publish = false; - self.publish_keyshare_created(ec) - } -} - -impl Actor for ThresholdKeyshare { - type Context = actix::Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT); - } -} - -impl ThresholdKeyshare { - pub fn ensure_collector( - &mut self, - self_addr: Addr, - ) -> Result> { - let Some(state) = self.state.get() else { - bail!("State not found on threshold keyshare. This should not happen."); - }; - - info!( - "Setting up key collector for addr: {} and {} nodes", - state.address, state.threshold_n - ); - let e3_id = state.e3_id.clone(); - let threshold_n = state.threshold_n; - let own_party_id = state.party_id; - let timeout = resolve_timeout( - DkgTimeoutPhase::ThresholdShareCollection, - state.dkg_started_at_unix_secs, - ); - info!( - e3_id = %e3_id, - timeout = ?timeout.duration, - "{}", - timeout.description - ); - let addr = self.decryption_key_collector.get_or_insert_with(|| { - ThresholdShareCollector::setup( - self_addr, - threshold_n, - own_party_id, - e3_id, - timeout.duration, - ) - }); - Ok(addr.clone()) - } - - pub fn ensure_encryption_key_collector( - &mut self, - self_addr: Addr, - ) -> Result> { - let Some(state) = self.state.get() else { - bail!("State not found on threshold keyshare. This should not happen."); - }; - - info!( - "Setting up encryption key collector for addr: {} and {} nodes", - state.address, state.threshold_n - ); - let e3_id = state.e3_id.clone(); - let threshold_n = state.threshold_n; - let timeout = resolve_timeout( - DkgTimeoutPhase::EncryptionKeyCollection, - state.dkg_started_at_unix_secs, - ); - info!( - e3_id = %e3_id, - timeout = ?timeout.duration, - "{}", - timeout.description - ); - let addr = self.encryption_key_collector.get_or_insert_with(|| { - EncryptionKeyCollector::setup(self_addr, threshold_n, e3_id, timeout.duration) - }); - Ok(addr.clone()) - } - - /// Create or return the DecryptionKeySharedCollector. - /// Uses honest_parties from persisted state. - pub fn ensure_decryption_key_shared_collector( - &mut self, - self_addr: Addr, - ) -> Result> { - let state = self.state.try_get()?; - let my_party_id = state.party_id; - - let honest = state - .honest_parties - .as_ref() - .ok_or_else(|| anyhow!("honest_parties not set when creating collector"))?; - - let expected: HashSet = honest - .iter() - .filter(|&&pid| pid != my_party_id) - .copied() - .collect(); - - let e3_id = state.e3_id.clone(); - let timeout = resolve_timeout( - DkgTimeoutPhase::DecryptionKeySharedCollection, - state.dkg_started_at_unix_secs, - ); - info!( - e3_id = %e3_id, - timeout = ?timeout.duration, - "{}", - timeout.description - ); - let addr = self.decryption_key_shared_collector.get_or_insert_with(|| { - DecryptionKeySharedCollector::setup(self_addr, expected, e3_id, timeout.duration) - }); - Ok(addr.clone()) - } - - fn handle_committee_member_expelled( - &mut self, - data: CommitteeMemberExpelled, - ec: EventContext, - ) { - // Only process enriched events (party_id resolved by Sortition). - // Raw events from chain (party_id = None) are ignored here; - // Sortition will re-publish them with party_id set. - let Some(party_id) = data.party_id else { - return; - }; - - let node_addr = data.node.to_string(); - info!( - "CommitteeMemberExpelled received (enriched): node={}, party_id={}, e3_id={}, active_count_after={}", - node_addr, party_id, data.e3_id, data.active_count_after - ); - - // Record permanently so late-arriving data is rejected even if - // collectors haven't been created or have already completed. - // Also clean honest_parties set for the expelled party. - let _ = self.state.try_mutate(&ec, |mut s| { - s.expelled_parties.insert(party_id); - if let Some(ref mut honest) = s.honest_parties { - honest.remove(&party_id); - } - Ok(s) - }); - - // Clean transient coordination state for the expelled party - self.pending_shares.retain(|s| s.party_id != party_id); - - if let Some(ref mut pending_c4) = self.pending_c4_verification_shares { - pending_c4.remove(&party_id); - } - - if let Some(ref collector) = self.encryption_key_collector { - collector.do_send(ExpelPartyFromKeyCollection { - party_id, - ec: ec.clone(), - }); - } - - if let Some(ref collector) = self.decryption_key_collector { - collector.do_send(ExpelPartyFromShareCollection { - party_id, - ec: ec.clone(), - }); - } - - if let Some(ref collector) = self.decryption_key_shared_collector { - collector.do_send(ExpelPartyFromDecryptionKeySharedCollection { party_id, ec }); - } - } - - pub fn handle_threshold_share_created( - &mut self, - msg: TypedEvent, - self_addr: Addr, - ) -> Result<()> { - let state = self.state.try_get()?; - if !matches!( - state.state, - KeyshareState::CollectingEncryptionKeys(_) - | KeyshareState::GeneratingThresholdShare(_) - | KeyshareState::AggregatingDecryptionKey(_) - ) { - trace!( - e3_id = %state.e3_id, - state = state.variant_name(), - sender_party_id = msg.share.party_id, - "Ignoring ThresholdShareCreated outside share collection" - ); - return Ok(()); - } - - let my_party_id = state.party_id; - - // Filter: only process shares intended for this party - if msg.target_party_id != my_party_id { - return Ok(()); - } - - // Reject shares from expelled parties - if state.expelled_parties.contains(&msg.share.party_id) { - info!( - "Dropping ThresholdShareCreated from expelled party {} for us (party {})", - msg.share.party_id, my_party_id - ); - return Ok(()); - } - - info!( - "Received ThresholdShareCreated from party {} for us (party {}), forwarding to collector!", - msg.share.party_id, my_party_id - ); - let collector = self.ensure_collector(self_addr)?; - info!("got collector address!"); - collector.do_send(msg); - Ok(()) - } - - pub fn handle_encryption_key_created( - &mut self, - msg: TypedEvent, - self_addr: Addr, - ) -> Result<()> { - let state = self.state.try_get()?; - if !matches!( - state.state, - KeyshareState::Init | KeyshareState::CollectingEncryptionKeys(_) - ) { - trace!( - e3_id = %state.e3_id, - state = state.variant_name(), - sender_party_id = msg.key.party_id, - "Ignoring EncryptionKeyCreated outside key collection" - ); - return Ok(()); - } - - // Reject keys from expelled parties - if state.expelled_parties.contains(&msg.key.party_id) { - info!( - "Dropping EncryptionKeyCreated from expelled party {}", - msg.key.party_id - ); - return Ok(()); - } - info!("Received EncryptionKeyCreated forwarding to encryption key collector!"); - let collector = self.ensure_encryption_key_collector(self_addr)?; - collector.do_send(msg); - Ok(()) - } - - /// Handle PkGenerationProofSigned - stores the signed C1 proof in state - pub fn handle_pk_generation_proof_signed( - &mut self, - msg: TypedEvent, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - let state = self.state.try_get()?; - - // Only accept proof for our own party - if msg.party_id != state.party_id { - return Ok(()); - } - - info!( - "Received PkGenerationProofSigned for party {} E3 {}", - msg.party_id, msg.e3_id - ); - - self.store_signed_pk_generation_proof(&ec, msg.signed_proof)?; - self.try_finish_deferred_keyshare_publish(ec)?; - - Ok(()) - } - - /// Handle DkgProofSigned - stores the signed proof in state based on proof type (C2a, C2b, C3a or C3b) - pub fn handle_share_computation_proof_signed( - &mut self, - msg: TypedEvent, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - let state = self.state.try_get()?; - - if msg.party_id != state.party_id { - return Ok(()); - } - - let proof_type = msg.signed_proof.payload.proof_type; - info!( - "Received DkgProofSigned ({:?}) for party {} E3 {}", - proof_type, msg.party_id, msg.e3_id - ); - - self.state.try_mutate(&ec, |s| { - let current: AggregatingDecryptionKey = s.clone().try_into()?; - let updated = match proof_type { - ProofType::C2aSkShareComputation => AggregatingDecryptionKey { - signed_sk_share_computation_proof: Some(msg.signed_proof), - ..current - }, - ProofType::C2bESmShareComputation => AggregatingDecryptionKey { - signed_e_sm_share_computation_proof: Some(msg.signed_proof), - ..current - }, - ProofType::C3aSkShareEncryption => { - let mut updated = current; - updated - .signed_sk_share_encryption_proofs - .push(msg.signed_proof); - updated - } - ProofType::C3bESmShareEncryption => { - let mut updated = current; - updated - .signed_e_sm_share_encryption_proofs - .push(msg.signed_proof); - updated - } - other => { - warn!("Unexpected proof type {:?} in DkgProofSigned", other); - current - } - }; - s.new_state(KeyshareState::AggregatingDecryptionKey(updated)) - })?; - - Ok(()) - } - - pub fn handle_compute_response( - &mut self, - msg: TypedEvent, - self_addr: Addr, - ) -> Result<()> { - match &msg.response { - ComputeResponseKind::TrBFV(trbfv) => match trbfv { - TrBFVResponse::GenEsiSss(_) => self.handle_gen_esi_sss_response(msg), - TrBFVResponse::GenPkShareAndSkSss(_) => { - self.handle_gen_pk_share_and_sk_sss_response(msg) - } - TrBFVResponse::CalculateDecryptionKey(_) => { - self.handle_calculate_decryption_key_response(msg, self_addr) - } - TrBFVResponse::CalculateDecryptionShare(_) => { - self.handle_calculate_decryption_share_response(msg) - } - _ => Ok(()), - }, - // ZK responses: proofs and verification are handled by - // ProofRequestActor and ShareVerificationActor respectively. - ComputeResponseKind::Zk(_) => Ok(()), - } - } - - /// 1. CiphernodeSelected - Generate BFV keys and publish EncryptionKeyPending - pub fn handle_ciphernode_selected( - &mut self, - msg: TypedEvent, - address: Addr, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - let state = self.state.try_get()?; - if !matches!(state.state, KeyshareState::Init) { - info!( - e3_id = %state.e3_id, - state = state.variant_name(), - "Ignoring replayed CiphernodeSelected; keyshare already initialized" - ); - return Ok(()); - } - - info!("CiphernodeSelected received."); - // Ensure the collectors are created - let _ = self.ensure_collector(address.clone()); - let _ = self.ensure_encryption_key_collector(address.clone()); - - let BfvKeypairMaterial { - sk_bfv: sk_bfv_encrypted, - pk_bfv: pk_bfv_bytes, - } = generate_bfv_keypair(&self.share_enc_preset, &self.cipher)?; - - let e3_id = state.e3_id.clone(); - - self.state.try_mutate(&ec, |s| { - s.new_state(KeyshareState::CollectingEncryptionKeys( - CollectingEncryptionKeysData { - sk_bfv: sk_bfv_encrypted.clone(), - pk_bfv: pk_bfv_bytes.clone(), - ciphernode_selected: msg, - }, - )) - })?; - - let committee_size = CiphernodesCommitteeSize::from_threshold( - state.threshold_m as usize, - state.threshold_n as usize, - )?; - self.bus.publish( - EncryptionKeyPending { - e3_id, - key: Arc::new(EncryptionKey::new(state.party_id, pk_bfv_bytes)), - params_preset: self.share_enc_preset, - committee_size, - }, - ec, - )?; - - Ok(()) - } - - /// 1a. AllEncryptionKeysCollected - All BFV keys received, start share generation - pub fn handle_all_encryption_keys_collected( - &mut self, - msg: TypedEvent, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - info!( - "AllEncryptionKeysCollected - {} keys received", - msg.keys.len() - ); - - let state = self.state.try_get()?; - let current: CollectingEncryptionKeysData = state.clone().try_into()?; - - // Filter out any keys from parties expelled after collection started - let filtered_keys: Vec<_> = if state.expelled_parties.is_empty() { - msg.keys - } else { - msg.keys - .into_iter() - .filter(|k| !state.expelled_parties.contains(&k.party_id)) - .collect() - }; - - self.state.try_mutate(&ec, |s| { - s.new_state(KeyshareState::GeneratingThresholdShare( - GeneratingThresholdShareData { - sk_sss: None, - pk_share: None, - esi_sss: None, - e_sm_raw: None, - sk_bfv: current.sk_bfv, - pk_bfv: current.pk_bfv, - collected_encryption_keys: filtered_keys, - ciphernode_selected: Some(current.ciphernode_selected.clone()), - proof_request_data: None, - }, - )) - })?; - - self.handle_gen_pk_share_and_sk_sss_requested(TypedEvent::new( - GenPkShareAndSkSss(current.ciphernode_selected), - ec, - ))?; - - Ok(()) - } - - /// 2. GenPkShareAndSkSss - pub fn handle_gen_pk_share_and_sk_sss_requested( - &self, - msg: TypedEvent, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - info!("GenPkShareAndSkSss on ThresholdKeyshare"); - let CiphernodeSelected { e3_id, .. } = msg.0; - let state = self - .state - .get() - .ok_or(anyhow!("State not found on ThrehsoldKeyshare"))?; - - let trbfv_config: TrBFVConfig = state.get_trbfv_config(); - - let crp = ArcBytes::from_bytes( - &create_deterministic_crp_from_default_seed(&trbfv_config.params()).to_bytes(), - ); - - let threshold_preset = self - .share_enc_preset - .threshold_counterpart() - .ok_or_else(|| anyhow!("No threshold counterpart for {:?}", self.share_enc_preset))?; - let defaults = threshold_preset - .search_defaults() - .ok_or_else(|| anyhow!("No search defaults for {:?}", threshold_preset))?; - - let event = ComputeRequest::trbfv( - TrBFVRequest::GenPkShareAndSkSss(GenPkShareAndSkSssRequest { - trbfv_config, - crp, - lambda: threshold_preset.lambda_config(), - num_ciphertexts: defaults.z as usize, - }), - CorrelationId::new(), - e3_id, - ); - - self.bus.publish(event, ec)?; - Ok(()) - } - - /// 2a. GenPkShareAndSkSss result - pub fn handle_gen_pk_share_and_sk_sss_response( - &mut self, - res: TypedEvent, - ) -> Result<()> { - let (res, ec) = res.into_components(); - - let output: GenPkShareAndSkSssResponse = res - .try_into() - .context("Error extracting data from compute process")?; - - let (pk_share, sk_sss, e_sm_raw) = ( - output.pk_share.clone(), - output.sk_sss, - output.e_sm_raw.clone(), - ); - - // Store proof request data for later use by ProofRequestActor - let proof_request_data = ProofRequestData { - pk0_share_raw: output.pk0_share_raw, - sk_raw: output.sk_raw, - eek_raw: output.eek_raw, - }; - - self.state.try_mutate(&ec, |s| { - info!("try_store_pk_share_and_sk_sss"); - let current: GeneratingThresholdShareData = s.clone().try_into()?; - s.new_state(KeyshareState::GeneratingThresholdShare( - GeneratingThresholdShareData { - pk_share: Some(pk_share), - sk_sss: Some(sk_sss), - e_sm_raw: Some(e_sm_raw.clone()), - proof_request_data: Some(proof_request_data), - ..current - }, - )) - })?; - - // Fire gen_esi_sss with the e_sm_raw - let current_state: GeneratingThresholdShareData = self.state.try_get()?.try_into()?; - if let Some(ciphernode_selected) = current_state.ciphernode_selected { - self.handle_gen_esi_sss_requested(TypedEvent::new( - GenEsiSss { - ciphernode_selected, - e_sm_raw: current_state - .e_sm_raw - .expect("e_sm_raw should be set at this point"), - }, - ec.clone(), - ))?; - } - - Ok(()) - } - - /// 3. GenEsiSss - pub fn handle_gen_esi_sss_requested(&self, msg: TypedEvent) -> Result<()> { - let (msg, ec) = msg.into_components(); - info!("GenEsiSss on ThresholdKeyshare"); - - let e_sm_raw = msg.e_sm_raw; - let CiphernodeSelected { e3_id, .. } = msg.ciphernode_selected; - - let state = self - .state - .get() - .ok_or(anyhow!("State not found on ThrehsoldKeyshare"))?; - - let trbfv_config = state.get_trbfv_config(); - - let event = ComputeRequest::trbfv( - TrBFVRequest::GenEsiSss(GenEsiSssRequest { - trbfv_config, - e_sm_raw, - }), - CorrelationId::new(), - e3_id, - ); - - self.bus.publish(event, ec)?; - Ok(()) - } - - /// 3a. GenEsiSss result - pub fn handle_gen_esi_sss_response(&mut self, res: TypedEvent) -> Result<()> { - let (res, ec) = res.into_components(); - let output: GenEsiSssResponse = res.try_into()?; - - let esi_sss = output.esi_sss; - - // First store esi_sss in GeneratingThresholdShareData - self.state.try_mutate(&ec, |s| { - info!("try_store_esi_sss"); - let current: GeneratingThresholdShareData = s.clone().try_into()?; - s.new_state(KeyshareState::GeneratingThresholdShare( - GeneratingThresholdShareData { - esi_sss: Some(esi_sss), - ..current - }, - )) - })?; - - info!("esi stored"); - - // Check if all data is ready, if so call handle_shares_generated BEFORE transitioning - let current: GeneratingThresholdShareData = self.state.try_get()?.try_into()?; - let ready = current.pk_share.is_some() - && current.sk_sss.is_some() - && current.esi_sss.is_some() - && current.e_sm_raw.is_some() - && current.proof_request_data.is_some(); - - if ready { - // Call handle_shares_generated while still in GeneratingThresholdShare state - self.handle_shares_generated(ec.clone())?; - - // Consume the own plaintext shares stashed transiently by handle_shares_generated. - let (own_sk_share_raw, own_esi_shares_raw) = - self.pending_own_dkg_shares.take().ok_or_else(|| { - anyhow!("pending_own_dkg_shares missing — handle_shares_generated did not run") - })?; - - // Now transition to AggregatingDecryptionKey with minimal state - self.state.try_mutate(&ec, |s| { - let current: GeneratingThresholdShareData = s.clone().try_into()?; - s.new_state(KeyshareState::AggregatingDecryptionKey( - AggregatingDecryptionKey { - pk_share: current.pk_share.expect("pk_share checked above"), - sk_bfv: current.sk_bfv, - own_sk_share_raw: own_sk_share_raw.clone(), - own_esi_shares_raw: own_esi_shares_raw.clone(), - signed_pk_generation_proof: None, - signed_sk_share_computation_proof: None, - signed_e_sm_share_computation_proof: None, - signed_sk_share_encryption_proofs: Vec::new(), - signed_e_sm_share_encryption_proofs: Vec::new(), - }, - )) - })?; - } - Ok(()) - } - - /// 4. SharesGenerated - Encrypt shares with BFV and publish - pub fn handle_shares_generated(&mut self, ec: EventContext) -> Result<()> { - let Some(ThresholdKeyshareState { - state: - KeyshareState::GeneratingThresholdShare(GeneratingThresholdShareData { - pk_share: Some(pk_share), - sk_sss: Some(sk_sss), - esi_sss: Some(esi_sss), - e_sm_raw: Some(e_sm_raw), - proof_request_data: Some(proof_request_data), - collected_encryption_keys, - .. - }), - party_id, - e3_id, - threshold_m, - threshold_n, - .. - }) = self.state.get() - else { - bail!("Invalid state - expected GeneratingThresholdShare with all data"); - }; - - // Decrypt our shares from local storage - let decrypted_sk_sss: SharedSecret = sk_sss.decrypt(&self.cipher)?; - let decrypted_esi_sss: Vec = esi_sss - .into_iter() - .map(|s| s.decrypt(&self.cipher)) - .collect::>()?; - - let plan = build_shares_generated_plan( - &self.cipher, - self.share_enc_preset, - party_id, - threshold_m, - threshold_n, - pk_share, - decrypted_sk_sss, - decrypted_esi_sss, - e_sm_raw, - proof_request_data, - &collected_encryption_keys, - )?; - - // Cache own plaintext share rows for the AggregatingDecryptionKey transition. - self.pending_own_dkg_shares = Some((plan.own_sk_share_raw, plan.own_esi_shares_raw)); - - let proof_aggregation_enabled = self - .state - .try_get() - .map(|s| s.proof_aggregation_enabled) - .unwrap_or(true); - - info!("Publishing ThresholdSharePending for E3 {}", e3_id); - - // Publish ThresholdSharePending - ProofRequestActor will generate proof, sign, and publish ThresholdShareCreated - self.bus.publish( - ThresholdSharePending { - e3_id, - full_share: Arc::new(plan.full_share), - proof_request: plan.proof_request, - sk_share_computation_request: plan.sk_share_computation_request, - e_sm_share_computation_request: plan.e_sm_share_computation_request, - sk_share_encryption_requests: plan.sk_share_encryption_requests, - e_sm_share_encryption_requests: plan.e_sm_share_encryption_requests, - recipient_party_ids: plan.recipient_party_ids, - proof_aggregation_enabled, - }, - ec, - )?; - - Ok(()) - } - - /// 5. AllThresholdSharesCollected - Verify C2/C3 proofs, then decrypt and aggregate - pub fn handle_all_threshold_shares_collected( - &mut self, - msg: TypedEvent, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - info!("AllThresholdSharesCollected"); - let state = self.state.try_get()?; - let e3_id = state.get_e3_id(); - let own_party_id = state.party_id; - - // Filter out expelled parties before any processing. The collector may - // have accepted shares before the expulsion arrived, so we scrub here. - let expelled = &state.expelled_parties; - let (shares, share_proofs): (Vec<_>, Vec<_>) = if expelled.is_empty() { - (msg.shares, msg.share_proofs) - } else { - warn!( - "Filtering {} expelled parties from AllThresholdSharesCollected for E3 {}: {:?}", - expelled.len(), - e3_id, - expelled - ); - msg.shares - .into_iter() - .zip(msg.share_proofs) - .filter(|(s, _)| !expelled.contains(&s.party_id)) - .unzip() - }; - - // Expected proof counts come from local cached own shares (trusted source); the - // collector excludes self from `shares`, so we cannot read them from there. - let current: AggregatingDecryptionKey = state.clone().try_into()?; - let own_sk_rows: Vec> = - bincode::deserialize(¤t.own_sk_share_raw.access_raw(&self.cipher)?) - .context("Failed to deserialize own_sk_share_raw")?; - let expected_c3a = own_sk_rows.len(); - let expected_num_esi = current.own_esi_shares_raw.len(); - let mut expected_c3b: usize = 0; - for esi_raw in current.own_esi_shares_raw.iter() { - let rows: Vec> = bincode::deserialize(&esi_raw.access_raw(&self.cipher)?) - .context("Failed to deserialize own esi share")?; - expected_c3b += rows.len(); - } - - // Build verification requests for other parties' proofs - let mut party_proofs_to_verify: Vec = Vec::new(); - let mut no_proof_parties: HashSet = HashSet::new(); - let mut incomplete_proof_parties: HashSet = HashSet::new(); - for (share, proofs) in shares.iter().zip(share_proofs.iter()) { - if share.party_id == own_party_id { - continue; - } - - let has_any_proof = proofs.signed_c2a_proof.is_some() - || proofs.signed_c2b_proof.is_some() - || !proofs.signed_c3a_proofs.is_empty() - || !proofs.signed_c3b_proofs.is_empty(); - - if !has_any_proof { - no_proof_parties.insert(share.party_id); - continue; - } - - // Validate proof set completeness against trusted expected counts. - // A malicious sender could omit proofs that would fail verification, - // so we must check that all expected proofs are present. - let is_complete = proofs.signed_c2a_proof.is_some() - && proofs.signed_c2b_proof.is_some() - && proofs.signed_c3a_proofs.len() == expected_c3a - && proofs.signed_c3b_proofs.len() == expected_c3b - && share.esi_sss.len() == expected_num_esi; - - if !is_complete { - warn!( - "Party {} has incomplete proof set (c2a={}, c2b={}, c3a={}/{}, c3b={}/{}, esi={}/{}), treating as dishonest", - share.party_id, - proofs.signed_c2a_proof.is_some(), - proofs.signed_c2b_proof.is_some(), - proofs.signed_c3a_proofs.len(), expected_c3a, - proofs.signed_c3b_proofs.len(), expected_c3b, - share.esi_sss.len(), expected_num_esi, - ); - incomplete_proof_parties.insert(share.party_id); - continue; - } - - // Complete proof set — collect for verification - let mut signed_proofs = Vec::new(); - // SAFETY: is_complete guarantees c2a and c2b are Some - signed_proofs.push(proofs.signed_c2a_proof.clone().unwrap()); - signed_proofs.push(proofs.signed_c2b_proof.clone().unwrap()); - signed_proofs.extend(proofs.signed_c3a_proofs.iter().cloned()); - signed_proofs.extend(proofs.signed_c3b_proofs.iter().cloned()); - - party_proofs_to_verify.push(PartyProofsToVerify { - sender_party_id: share.party_id, - signed_proofs, - }); - } - - // Store shares on the actor for use after verification completes (keep Arc to avoid deep clone) - self.pending_shares = shares.to_vec(); - - // Merge no-proof and incomplete-proof parties — both are dishonest - let mut pre_dishonest: BTreeSet = BTreeSet::new(); - pre_dishonest.extend(incomplete_proof_parties); - pre_dishonest.extend(no_proof_parties); - if !pre_dishonest.is_empty() { - warn!( - "{} parties have missing/incomplete C2/C3 proofs for E3 {} — marking as pre-dishonest: {:?}", - pre_dishonest.len(), - e3_id, - pre_dishonest - ); - } - - if party_proofs_to_verify.is_empty() { - // All non-self parties are dishonest (missing or incomplete proofs), none to verify - let threshold = state.threshold_m; - let total = state.threshold_n; - let dishonest_count = (pre_dishonest.len() as u64).min(total); - let honest_count = total - dishonest_count; - - if honest_count <= threshold { - warn!( - "Too few honest parties for E3 {} ({} honest, need at least {}) after C2/C3 pre-dishonest filtering — cannot proceed", - e3_id, honest_count, threshold + 1 - ); - self.pending_shares.clear(); - self.bus.publish( - E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::InsufficientCommitteeMembers, - }, - ec, - )?; - return Ok(()); - } - - let dishonest_set: HashSet = pre_dishonest.into_iter().collect(); - return self.proceed_with_decryption_key_calculation(Some(dishonest_set), ec); - } - - info!( - "Dispatching C2/C3 share verification for E3 {} ({} parties, {} pre-dishonest)", - e3_id, - party_proofs_to_verify.len(), - pre_dishonest.len() - ); - - let committee_size = CiphernodesCommitteeSize::from_threshold( - state.threshold_m as usize, - state.threshold_n as usize, - )?; - self.bus.publish( - ShareVerificationDispatched { - e3_id: e3_id.clone(), - kind: VerificationKind::ShareProofs, - share_proofs: party_proofs_to_verify, - decryption_proofs: Vec::new(), - pre_dishonest, - params_preset: self.share_enc_preset, - committee_size, - }, - ec, - )?; - Ok(()) - } - - /// Handle ShareVerificationComplete from ShareVerificationActor. - /// Dispatched for both C2/C3 and C4 verification. - pub fn handle_share_verification_complete( - &mut self, - msg: TypedEvent, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - let state = self.state.try_get()?; - let e3_id = state.get_e3_id(); - - match msg.kind { - VerificationKind::ShareProofs => { - // C2/C3 verification complete - if msg.dishonest_parties.is_empty() { - info!( - "All parties passed C2/C3 verification for E3 {} — proceeding", - e3_id - ); - self.proceed_with_decryption_key_calculation(None, ec) - } else { - let threshold = state.threshold_m; - let total = state.threshold_n; - let dishonest_count = (msg.dishonest_parties.len() as u64).min(total); - let honest_count = total - dishonest_count; - - if honest_count <= threshold { - warn!( - "Too few honest parties for E3 {} ({} honest, need at least {}) — cannot proceed", - e3_id, honest_count, threshold + 1 - ); - // Clear pending shares - self.pending_shares.clear(); - self.bus.publish( - E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::InsufficientCommitteeMembers, - }, - ec, - )?; - return Ok(()); - } - - let dishonest_set: HashSet = msg.dishonest_parties.into_iter().collect(); - info!( - "Proceeding with {} honest parties for E3 {} ({} dishonest excluded)", - honest_count, - e3_id, - dishonest_set.len() - ); - self.proceed_with_decryption_key_calculation(Some(dishonest_set), ec) - } - } - VerificationKind::DecryptionProofs => { - // C4 verification complete — update honest set and publish KeyshareCreated - if !msg.dishonest_parties.is_empty() { - self.state.try_mutate(&ec, |mut s| { - if let Some(ref mut honest) = s.honest_parties { - honest.retain(|pid| !msg.dishonest_parties.contains(pid)); - } - Ok(s) - })?; - - let state = self.state.try_get()?; - let threshold = state.threshold_m; - let honest_count = state - .honest_parties - .as_ref() - .map(|h| h.len() as u64) - .unwrap_or(0); - - if honest_count <= threshold { - warn!( - "Too few honest parties after C4 for E3 {} ({} honest, need at least {})", - e3_id, honest_count, threshold + 1 - ); - self.bus.publish( - E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::InsufficientCommitteeMembers, - }, - ec, - )?; - return Ok(()); - } - - info!( - "Updated honest set after C4 for E3 {}: {} honest ({} removed)", - e3_id, - honest_count, - msg.dishonest_parties.len() - ); - } else { - info!( - "All parties passed C4 verification for E3 {} — publishing KeyshareCreated", - e3_id - ); - } - - self.publish_keyshare_created(ec) - } - _ => Ok(()), - } - } - - /// After verification, decrypt shares from honest parties and compute decryption key. - /// C4 proof generation is deferred to ProofRequestActor via DecryptionShareProofsPending. - fn proceed_with_decryption_key_calculation( - &mut self, - dishonest_parties: Option>, - ec: EventContext, - ) -> Result<()> { - let state = self.state.try_get()?; - let e3_id = state.get_e3_id(); - let trbfv_config = state.get_trbfv_config(); - - // Get our BFV secret key from state, pending shares from the actor - let current: AggregatingDecryptionKey = state.clone().try_into()?; - let shares = std::mem::take(&mut self.pending_shares); - - let plan = build_decryption_key_plan( - &self.cipher, - self.share_enc_preset, - state.party_id, - state.threshold_m, - state.threshold_n, - trbfv_config, - ¤t, - shares, - dishonest_parties, - e3_id, - )?; - - match plan { - DecryptionKeyPlan::Insufficient => { - self.pending_shares.clear(); - self.bus.publish( - E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::InsufficientCommitteeMembers, - }, - ec, - )?; - } - DecryptionKeyPlan::Proceed { - calc_request, - sk_request, - esm_requests, - honest_party_ids, - } => { - // Publish CalculateDecryptionKey request before persisting (ordering preserved). - let event = ComputeRequest::trbfv( - TrBFVRequest::CalculateDecryptionKey(calc_request), - CorrelationId::new(), - e3_id.clone(), - ); - self.bus.publish(event, ec.clone())?; - - // Store honest parties and C4 data on the actor (transient coordination) - self.state.try_mutate(&ec, |mut s| { - s.honest_parties = Some(honest_party_ids.clone()); - Ok(s) - })?; - self.pending_share_decryption_data = Some((sk_request, esm_requests)); - } - } - - Ok(()) - } - - /// 5a. CalculateDecryptionKeyResponse — transition to ReadyForDecryption, - /// then publish DecryptionShareProofsPending so ProofRequestActor can - /// generate C4 proofs, sign them, and publish DecryptionKeyShared. - pub fn handle_calculate_decryption_key_response( - &mut self, - res: TypedEvent, - self_addr: Addr, - ) -> Result<()> { - let (res, ec) = res.into_components(); - let output: CalculateDecryptionKeyResponse = res - .try_into() - .context("Error extracting data from compute process")?; - - let (sk_poly_sum, es_poly_sum) = (output.sk_poly_sum, output.es_poly_sum); - - // Extract C4 data from the actor (stored by proceed_with_decryption_key_calculation) - let (sk_request, esm_requests) = self - .pending_share_decryption_data - .take() - .ok_or_else(|| anyhow!("No pending share decryption data — CalculateDecryptionKey responded before proof requests were built"))?; - - // Take early shares from the actor before transitioning - let early_shares = self - .pending_c4_verification_shares - .take() - .unwrap_or_default(); - - // Transition to ReadyForDecryption - self.state.try_mutate(&ec, |s| { - use KeyshareState as K; - info!("Try store decryption key"); - - let current: AggregatingDecryptionKey = s.clone().try_into()?; - - let next = K::ReadyForDecryption(ReadyForDecryption { - pk_share: current.pk_share, - sk_poly_sum, - es_poly_sum, - signed_pk_generation_proof: current.signed_pk_generation_proof, - signed_sk_share_computation_proof: current.signed_sk_share_computation_proof, - signed_e_sm_share_computation_proof: current.signed_e_sm_share_computation_proof, - signed_sk_share_encryption_proofs: current.signed_sk_share_encryption_proofs, - signed_e_sm_share_encryption_proofs: current.signed_e_sm_share_encryption_proofs, - }); - - s.new_state(next) - })?; - - // Publish DecryptionShareProofsPending to ProofRequestActor - let state = self.state.try_get()?; - let e3_id = state.get_e3_id(); - let party_id = state.party_id; - let node = state.address.clone(); - - info!( - "Publishing DecryptionShareProofsPending for E3 {} party {} (1 SK + {} ESM requests)", - e3_id, - party_id, - esm_requests.len() - ); - - self.bus.publish( - DecryptionShareProofsPending { - e3_id: e3_id.clone(), - party_id, - node, - sk_request, - esm_requests, - }, - ec.clone(), - )?; - - // Create collector and replay any early-arriving DecryptionKeyShared events - let state = self.state.try_get()?; - let my_party_id = state.party_id; - let honest = state.honest_parties.as_ref().cloned().unwrap_or_default(); - let expected: HashSet = honest - .iter() - .filter(|&&pid| pid != my_party_id) - .copied() - .collect(); - - if !expected.is_empty() { - let collector = self.ensure_decryption_key_shared_collector(self_addr)?; - for (_pid, share) in early_shares { - collector.do_send(TypedEvent::new(share, ec.clone())); - } - } - - Ok(()) - } - - /// Handle an external DecryptionKeyShared event while in AggregatingDecryptionKey state. - /// Store it for later processing when we transition to ReadyForDecryption. - fn handle_early_decryption_key_share( - &mut self, - data: DecryptionKeyShared, - _ec: EventContext, - ) -> Result<()> { - let party_id = data.party_id; - let state = self.state.try_get()?; - if state.expelled_parties.contains(&party_id) { - info!( - "Dropping early DecryptionKeyShared from expelled party {}", - party_id - ); - return Ok(()); - } - info!( - "Storing early DecryptionKeyShared from party {} (state: AggregatingDecryptionKey)", - party_id - ); - self.pending_c4_verification_shares - .get_or_insert_with(HashMap::new) - .insert(party_id, data); - Ok(()) - } - - /// Dispatch C4 verification for all collected DecryptionKeyShared events. - /// Shares are provided by the DecryptionKeySharedCollector. - fn dispatch_c4_verification( - &mut self, - collected_shares: HashMap, - ec: EventContext, - ) -> Result<()> { - let state = self.state.try_get()?; - let e3_id = state.get_e3_id(); - let ready: ReadyForDecryption = state.clone().try_into()?; - - info!( - "All DecryptionKeyShared collected for E3 {} ({} shares)", - e3_id, - collected_shares.len() - ); - - // Validate ESM proof count — each party must provide exactly - // one C4b proof per smudging noise index. - let expected_esm = ready.es_poly_sum.len(); - let mut c4_count_dishonest: HashSet = HashSet::new(); - let party_proofs: Vec = collected_shares - .iter() - .filter_map(|(&party_id, share)| { - if share.signed_e_sm_decryption_proofs.len() != expected_esm { - warn!( - "Party {} has wrong ESM proof count ({} vs expected {}) for E3 {} — treating as dishonest", - party_id, - share.signed_e_sm_decryption_proofs.len(), - expected_esm, - e3_id - ); - c4_count_dishonest.insert(party_id); - None - } else { - Some(PartyShareDecryptionProofsToVerify { - sender_party_id: party_id, - signed_sk_decryption_proof: share.signed_sk_decryption_proof.clone(), - signed_e_sm_decryption_proofs: share.signed_e_sm_decryption_proofs.clone(), - }) - } - }) - .collect(); - - // Evict pre-dishonest parties (wrong ESM count) from honest set - if !c4_count_dishonest.is_empty() { - self.state.try_mutate(&ec, |mut s| { - if let Some(ref mut honest) = s.honest_parties { - honest.retain(|pid| !c4_count_dishonest.contains(pid)); - } - Ok(s) - })?; - } - - if party_proofs.is_empty() { - // Check threshold viability after removing pre-dishonest parties - let state = self.state.try_get()?; - let threshold = state.threshold_m; - let honest_count = state - .honest_parties - .as_ref() - .map(|h| h.len() as u64) - .unwrap_or(0); - - if honest_count <= threshold { - warn!( - "Too few honest parties after C4 pre-filtering for E3 {} ({} honest, need at least {})", - e3_id, honest_count, threshold + 1 - ); - self.bus.publish( - E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::InsufficientCommitteeMembers, - }, - ec, - )?; - return Ok(()); - } - - info!("No C4 proofs to verify — publishing KeyshareCreated directly"); - return self.publish_keyshare_created(ec); - } - - let pre_dishonest: BTreeSet = c4_count_dishonest.into_iter().collect(); - - info!( - "Dispatching C4 share verification for E3 {} ({} parties, {} pre-dishonest)", - e3_id, - party_proofs.len(), - pre_dishonest.len() - ); - - let committee_size = CiphernodesCommitteeSize::from_threshold( - state.threshold_m as usize, - state.threshold_n as usize, - )?; - self.bus.publish( - ShareVerificationDispatched { - e3_id: e3_id.clone(), - kind: VerificationKind::DecryptionProofs, - share_proofs: Vec::new(), - decryption_proofs: party_proofs, - pre_dishonest, - params_preset: self.share_enc_preset, - committee_size, - }, - ec, - )?; - Ok(()) - } - - /// Publish KeyshareCreated (Exchange #4) with pk_share and signed C1 proof. - fn publish_keyshare_created(&mut self, ec: EventContext) -> Result<()> { - let state = self.state.try_get()?; - let e3_id = state.get_e3_id(); - let address = state.get_address().to_owned(); - let party_id = state.get_party_id(); - let Some((pk_share, signed_pk_generation_proof)) = - Self::keyshare_created_fields(&state.state) - else { - warn!( - "Deferring KeyshareCreated for party {} E3 {} — not in ReadyForDecryption/Decrypting ({})", - party_id, - e3_id, - state.state.variant_name() - ); - self.pending_keyshare_publish = true; - return Ok(()); - }; - - if signed_pk_generation_proof.is_none() { - warn!( - "Deferring KeyshareCreated for party {} E3 {} — C1 proof not stored yet (PkGenerationProofSigned race)", - party_id, e3_id - ); - self.pending_keyshare_publish = true; - return Ok(()); - } - - info!("Publishing Exchange #4 (KeyshareCreated) for E3 {}", e3_id); - - self.bus.publish( - KeyshareCreated { - pubkey: pk_share.clone(), - e3_id: e3_id.clone(), - node: address, - party_id, - signed_pk_generation_proof: signed_pk_generation_proof.clone(), - }, - ec.clone(), - )?; - - // Record that publishing was authorized and has occurred, so resume-after-crash - // may safely re-publish (idempotent at the aggregator) without ever emitting a - // keyshare for a state that had not yet passed C4 honest-set filtering. - self.state.try_mutate(&ec, |mut s| { - s.keyshare_published = true; - Ok(s) - })?; - - Ok(()) - } - - /// CiphertextOutputPublished - pub fn handle_ciphertext_output_published( - &mut self, - msg: TypedEvent, - ) -> Result<()> { - let (msg, ec) = msg.into_components(); - let ciphertext_output = msg.ciphertext_output; - - // If we are already in Decrypting (or beyond), this is a duplicate - // event — e.g. replayed from the EventStore during crash recovery. - // Re-issue the compute request idempotently (downstream aggregation - // deduplicates by party_id) and skip the state transition. - { - let state = self.state.try_get()?; - match &state.state { - KeyshareState::Decrypting(_) => { - info!( - e3_id = %state.e3_id, - "CiphertextOutputPublished received while already Decrypting — re-issuing decryption-share request" - ); - return self.issue_decryption_share_request(ec); - } - KeyshareState::GeneratingDecryptionProof(_) | KeyshareState::Completed => { - info!( - e3_id = %state.e3_id, - state = %state.variant_name(), - "CiphertextOutputPublished received after decryption already in progress — ignoring" - ); - return Ok(()); - } - KeyshareState::ReadyForDecryption(_) => { - // Normal path — proceed with transition below. - } - other => { - warn!( - e3_id = %state.e3_id, - state = %other.variant_name(), - "CiphertextOutputPublished received in unexpected state — cannot process decryption" - ); - return Ok(()); - } - } - } - - // Set state to decrypting, storing ciphertext for later C6 proof generation - self.state.try_mutate(&ec, |s| { - use KeyshareState as K; - - let current: ReadyForDecryption = s.clone().try_into()?; - - let next = K::Decrypting(Decrypting { - pk_share: current.pk_share, - sk_poly_sum: current.sk_poly_sum, - es_poly_sum: current.es_poly_sum, - ciphertext_output: ciphertext_output.clone(), - signed_pk_generation_proof: current.signed_pk_generation_proof, - signed_sk_share_computation_proof: current.signed_sk_share_computation_proof, - signed_e_sm_share_computation_proof: current.signed_e_sm_share_computation_proof, - signed_sk_share_encryption_proofs: current.signed_sk_share_encryption_proofs, - signed_e_sm_share_encryption_proofs: current.signed_e_sm_share_encryption_proofs, - }); - - s.new_state(next) - })?; - - let state = self.state.try_get()?; - let e3_id = state.get_e3_id(); - let decrypting: Decrypting = state.clone().try_into()?; - let trbfv_config = state.get_trbfv_config(); - let event = ComputeRequest::trbfv( - TrBFVRequest::CalculateDecryptionShare(CalculateDecryptionShareRequest { - name: format!("party_id({})", state.party_id), - ciphertexts: ciphertext_output, - sk_poly_sum: decrypting.sk_poly_sum, - es_poly_sum: decrypting.es_poly_sum, - trbfv_config, - }), - CorrelationId::new(), - e3_id.clone(), - ); - self.bus.publish(event, ec)?; // CalculateDecryptionShareRequest - Ok(()) - } - - /// (Re)issue the `CalculateDecryptionShare` compute request from the current - /// `Decrypting` state. Factored out of `handle_ciphertext_output_published` so the - /// boot-time resume path can re-drive the decryption-share computation idempotently - /// (the resulting `DecryptionshareCreated` is deduped by `party_id` at the aggregator). - fn issue_decryption_share_request(&self, ec: EventContext) -> Result<()> { - let state = self.state.try_get()?; - let e3_id = state.get_e3_id(); - let decrypting: Decrypting = state.clone().try_into()?; - let trbfv_config = state.get_trbfv_config(); - let event = ComputeRequest::trbfv( - TrBFVRequest::CalculateDecryptionShare(CalculateDecryptionShareRequest { - name: format!("party_id({})", state.party_id), - ciphertexts: decrypting.ciphertext_output, - sk_poly_sum: decrypting.sk_poly_sum, - es_poly_sum: decrypting.es_poly_sum, - trbfv_config, - }), - CorrelationId::new(), - e3_id.clone(), - ); - self.bus.publish(event, ec)?; - Ok(()) - } - - /// Re-drive this node's own in-flight DKG/decryption work after a crash or restart. - /// - /// Invoked when `EffectsEnabled` is broadcast at the end of boot sync, *after* this - /// actor has been hydrated from its persisted state. The dangerous, value-bearing - /// concern with re-driving is double-emission; this is safe here because every output - /// re-emitted below is deduplicated downstream by `party_id`: - /// * `KeyshareCreated` — `PublicKeyAggregation::add_keyshare` ignores a `party_id` - /// that already submitted (idempotent), and - /// * `DecryptionshareCreated` — threshold plaintext aggregation keys shares by - /// `party_id` (re-insert overwrites with the identical deterministic share). - /// - /// Only states where the local result is already determined are re-driven. Earlier - /// phases depend on peer gossip that cannot be reconstructed locally and are surfaced - /// (non-destructively) by `interfold node validate` instead of being force-re-driven. - fn resume_in_flight_work(&mut self, ec: EventContext) -> Result<()> { - let Some(state) = self.state.get() else { - return Ok(()); - }; - match &state.state { - // We have produced our public-key share but may have crashed before (or while) - // publishing KeyshareCreated. Re-publishing is idempotent at the aggregator, but - // ReadyForDecryption is entered *before* C4 honest-set verification authorizes the - // publish, so only re-drive when a prior authorized publish was recorded. An - // un-published ReadyForDecryption is a loose end surfaced by `interfold node validate`. - KeyshareState::ReadyForDecryption(_) if state.keyshare_published => { - info!( - e3_id = %state.e3_id, - "Resuming in-flight work: re-publishing KeyshareCreated" - ); - self.publish_keyshare_created(ec)?; - } - // The ciphertext to decrypt has arrived. Re-publish our keyshare (in case the - // crash happened before it propagated) and re-issue the decryption-share - // computation so a DecryptionshareCreated is (re)produced. - KeyshareState::Decrypting(_) => { - info!( - e3_id = %state.e3_id, - "Resuming in-flight work: re-publishing KeyshareCreated and re-issuing decryption-share request" - ); - self.publish_keyshare_created(ec.clone())?; - self.issue_decryption_share_request(ec)?; - } - other => { - trace!( - e3_id = %state.e3_id, - state = %other.variant_name(), - "No locally re-drivable work on resume; loose ends are surfaced by `interfold node validate`" - ); - } - } - Ok(()) - } - - /// CalculateDecryptionShareResponse — publish ShareDecryptionProofPending - /// so ProofRequestActor generates and signs C6 proofs. - pub fn handle_calculate_decryption_share_response( - &mut self, - res: TypedEvent, - ) -> Result<()> { - let (res, ec) = res.into_components(); - let msg: CalculateDecryptionShareResponse = res.try_into()?; - let state = self.state.try_get()?; - let e3_id = state.e3_id.clone(); - let decrypting: Decrypting = state.clone().try_into()?; - let d_share_poly = msg.d_share_poly; - - let aggregated_pk_bytes = state - .aggregated_pk - .clone() - .ok_or_else(|| anyhow!("Aggregated public key not available for C6 proof"))?; - - let threshold_preset = self - .share_enc_preset - .threshold_counterpart() - .ok_or_else(|| { - anyhow!( - "No threshold counterpart for preset {:?}", - self.share_enc_preset - ) - })?; - - info!("Publishing ShareDecryptionProofPending for C6 proof generation..."); - - let committee_size = CiphernodesCommitteeSize::from_threshold( - state.threshold_m as usize, - state.threshold_n as usize, - )?; - - // Publish pending event before transitioning state so a publish - // failure leaves us in Decrypting (retryable) rather than - // GeneratingDecryptionProof (no retry path). - self.bus.publish( - ShareDecryptionProofPending { - e3_id: e3_id.clone(), - party_id: state.party_id, - node: state.address.clone(), - decryption_share: d_share_poly.clone(), - proof_request: ThresholdShareDecryptionProofRequest { - ciphertext_bytes: decrypting.ciphertext_output, - aggregated_pk_bytes, - sk_poly_sum: decrypting.sk_poly_sum, - es_poly_sum: decrypting.es_poly_sum, - d_share_bytes: d_share_poly.clone(), - params_preset: threshold_preset, - committee_size, - }, - }, - ec.clone(), - )?; - - // Transition to GeneratingDecryptionProof state - self.state.try_mutate(&ec, |s| { - use KeyshareState as K; - s.new_state(K::GeneratingDecryptionProof(GeneratingDecryptionProof { - pk_share: decrypting.pk_share.clone(), - decryption_share: d_share_poly, - signed_pk_generation_proof: decrypting.signed_pk_generation_proof.clone(), - signed_sk_share_computation_proof: decrypting - .signed_sk_share_computation_proof - .clone(), - signed_e_sm_share_computation_proof: decrypting - .signed_e_sm_share_computation_proof - .clone(), - signed_sk_share_encryption_proofs: decrypting - .signed_sk_share_encryption_proofs - .clone(), - signed_e_sm_share_encryption_proofs: decrypting - .signed_e_sm_share_encryption_proofs - .clone(), - })) - })?; - Ok(()) - } - - pub fn handle_decryption_share_proof_signed( - &mut self, - msg: TypedEvent, - ) -> Result<()> { - let (_msg, ec) = msg.into_components(); - - self.state.try_mutate(&ec, |s| { - use KeyshareState as K; - info!("Decryption share sending process is complete"); - s.new_state(K::Completed) - })?; - - Ok(()) - } -} - -// Will only receive events that are for this specific e3_id -impl Handler for ThresholdKeyshare { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - match msg { - InterfoldEventData::CiphernodeSelected(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::CiphertextOutputPublished(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::PublicKeyAggregated(data) => { - let pk = ArcBytes::from_bytes(&data.pubkey); - let _ = self.state.try_mutate(&ec, |mut s| { - s.aggregated_pk = Some(pk); - Ok(s) - }); - } - InterfoldEventData::ThresholdShareCreated(data) => { - let _ = - self.handle_threshold_share_created(TypedEvent::new(data, ec), ctx.address()); - } - InterfoldEventData::EncryptionKeyCreated(data) => { - let _ = - self.handle_encryption_key_created(TypedEvent::new(data, ec), ctx.address()); - } - InterfoldEventData::PkGenerationProofSigned(data) => { - let _ = self.handle_pk_generation_proof_signed(TypedEvent::new(data, ec)); - } - InterfoldEventData::DkgProofSigned(data) => { - let _ = self.handle_share_computation_proof_signed(TypedEvent::new(data, ec)); - } - InterfoldEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), - InterfoldEventData::E3Failed(data) => { - warn!( - "E3 failed: {:?}. Shutting down ThresholdKeyshare for e3_id={}", - data.reason, data.e3_id - ); - self.notify_sync(ctx, E3RequestComplete { e3_id: data.e3_id }); - } - InterfoldEventData::E3StageChanged(data) => { - use e3_events::E3Stage; - match &data.new_stage { - E3Stage::Complete | E3Stage::Failed => { - info!("E3 reached terminal stage {:?}. Shutting down ThresholdKeyshare for e3_id={}", data.new_stage, data.e3_id); - self.notify_sync(ctx, E3RequestComplete { e3_id: data.e3_id }); - } - _ => { - trace!( - "E3 stage changed to {:?} for e3_id={}", - data.new_stage, - data.e3_id - ); - } - } - } - InterfoldEventData::DecryptionKeyShared(data) => { - if data.external { - // Route based on current state - if let Some(state) = self.state.get() { - if state.expelled_parties.contains(&data.party_id) { - info!( - "Dropping DecryptionKeyShared from expelled party {}", - data.party_id - ); - return; - } - let result = match &state.state { - KeyshareState::AggregatingDecryptionKey(_) => { - self.handle_early_decryption_key_share(data, ec) - } - KeyshareState::ReadyForDecryption(_) => { - // Delegate to the collector actor - if let Some(ref collector) = self.decryption_key_shared_collector { - collector.do_send(TypedEvent::new(data, ec)); - Ok(()) - } else { - warn!( - "DecryptionKeyShared from party {} dropped — no collector (sole honest party)", - data.party_id - ); - Ok(()) - } - } - other => { - trace!( - "DecryptionKeyShared from party {} in unexpected state {:?}, ignoring", - data.party_id, - other.variant_name() - ); - Ok(()) - } - }; - if let Err(err) = result { - error!("Failed to handle DecryptionKeyShared: {err}"); - } - } - } else { - // Own DecryptionKeyShared published by ProofRequestActor. - // A3 fast-path: if no other honest parties, publish KeyshareCreated directly. - if let Some(state) = self.state.get() { - if data.party_id == state.party_id { - if let KeyshareState::ReadyForDecryption(_) = state.state { - let others = state - .honest_parties - .as_ref() - .map(|h| h.iter().filter(|&&pid| pid != state.party_id).count()) - .unwrap_or(0); - if others == 0 { - info!( - "No other honest parties for E3 {} — publishing KeyshareCreated directly", - data.e3_id - ); - if let Err(err) = self.publish_keyshare_created(ec) { - error!("Failed to publish KeyshareCreated: {err}"); - } - } - } - } - } - } - } - InterfoldEventData::DecryptionShareProofSigned(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ShareVerificationComplete(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ComputeResponse(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::CommitteeMemberExpelled(data) => { - self.handle_committee_member_expelled(data, ec); - } - InterfoldEventData::EffectsEnabled(_) => { - // Broadcast once at the end of boot sync. Re-drive any of this node's own - // in-flight work that a crash may have interrupted (idempotent downstream). - if let Err(err) = self.resume_in_flight_work(ec) { - warn!("resume_in_flight_work failed: {err}"); - } - } - _ => (), - } - } -} - -impl Handler> for ThresholdKeyshare { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::KeyGeneration, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_decryption_share_proof_signed(msg), - ) - } -} - -impl Handler> for ThresholdKeyshare { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::KeyGeneration, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_compute_response(msg, ctx.address()), - ) - } -} - -impl Handler> for ThresholdKeyshare { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::KeyGeneration, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_ciphernode_selected(msg, ctx.address()), - ) - } -} - -impl Handler> for ThresholdKeyshare { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - trap( - EType::KeyGeneration, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_all_encryption_keys_collected(msg), - ) - } -} - -impl Handler> for ThresholdKeyshare { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - trap( - EType::KeyGeneration, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_share_verification_complete(msg), - ) - } -} - -impl Handler> for ThresholdKeyshare { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - trap( - EType::KeyGeneration, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_all_threshold_shares_collected(msg), - ) - } -} - -impl Handler> for ThresholdKeyshare { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - trap( - EType::KeyGeneration, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_ciphertext_output_published(msg), - ) - } -} - -impl Handler for ThresholdKeyshare { - type Result = (); - fn handle( - &mut self, - msg: EncryptionKeyCollectionFailed, - ctx: &mut Self::Context, - ) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - warn!( - e3_id = %msg.e3_id, - missing_parties = ?msg.missing_parties, - "Encryption key collection failed: {}", - msg.reason - ); - - // Clear the collector reference since it's stopped - self.encryption_key_collector = None; - - // Publish failure event to event bus for sync tracking - self.bus.publish_without_context(msg.clone())?; - - self.bus.publish_without_context(E3Failed { - e3_id: msg.e3_id, - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DKGTimeout, - })?; - - // Stop this actor since we can't proceed without all encryption keys - ctx.stop(); - Ok(()) - }) - } -} - -impl Handler for ThresholdKeyshare { - type Result = (); - fn handle( - &mut self, - msg: ThresholdShareCollectionFailed, - ctx: &mut Self::Context, - ) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - warn!( - e3_id = %msg.e3_id, - missing_parties = ?msg.missing_parties, - "Threshold share collection failed: {}", - msg.reason - ); - - // Clear the collector reference since it's stopped - self.decryption_key_collector = None; - - // Publish failure event to event bus for sync tracking - self.bus.publish_without_context(msg.clone())?; - - self.bus.publish_without_context(E3Failed { - e3_id: msg.e3_id, - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DKGTimeout, - })?; - - ctx.stop(); - Ok(()) - }) - } -} - -impl Handler> for ThresholdKeyshare { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - trap( - EType::KeyGeneration, - &self.bus.with_ec(msg.get_ctx()), - || { - let (msg, ec) = msg.into_components(); - self.decryption_key_shared_collector = None; - self.dispatch_c4_verification(msg.shares, ec) - }, - ) - } -} - -impl Handler for ThresholdKeyshare { - type Result = (); - fn handle( - &mut self, - msg: DecryptionKeySharedCollectionFailed, - ctx: &mut Self::Context, - ) -> Self::Result { - trap(EType::KeyGeneration, &self.bus.clone(), || { - warn!( - e3_id = %msg.e3_id, - missing_parties = ?msg.missing_parties, - "DecryptionKeyShared collection failed: {}", - msg.reason - ); - - self.decryption_key_shared_collector = None; - - self.bus.publish_without_context(E3Failed { - e3_id: msg.e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DecryptionTimeout, - })?; - - ctx.stop(); - Ok(()) - }) - } -} - -impl Handler for ThresholdKeyshare { - type Result = (); - fn handle(&mut self, _: E3RequestComplete, ctx: &mut Self::Context) -> Self::Result { - self.encryption_key_collector = None; - self.decryption_key_collector = None; - self.decryption_key_shared_collector = None; - self.pending_shares.clear(); - self.pending_share_decryption_data = None; - self.pending_c4_verification_shares = None; - self.pending_keyshare_publish = false; - self.notify_sync(ctx, Die); - } -} - -impl Handler for ThresholdKeyshare { - type Result = (); - fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { - warn!("ThresholdKeyshare is shutting down"); - ctx.stop(); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::actors::decryption_key_shared_collector::DecryptionKeySharedCollectionFailed; - use actix::{Actor, Addr, Handler}; - use anyhow::Result; - use e3_crypto::Cipher; - use e3_data::{AutoPersist, DataStore, InMemStore, Persistable, Repository}; - use e3_events::{ - hlc_factory::HlcFactory, BusHandle, E3Stage, E3id, EventBus, EventBusConfig, FailureReason, - HistoryCollector, InterfoldEvent, InterfoldEventData, Sequencer, StoreEventRequested, - StoreEventResponse, TakeEvents, - }; - use e3_fhe_params::DEFAULT_BFV_PRESET; - use std::sync::Arc; - - #[derive(Default)] - struct TestEventStore { - next_seq: u64, - } - - impl Actor for TestEventStore { - type Context = actix::Context; - } - - impl Handler for TestEventStore { - type Result = (); - - fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { - let StoreEventRequested { event, sender } = msg; - let seq = self.next_seq; - self.next_seq += 1; - sender.do_send(StoreEventResponse(event.into_sequenced(seq))); - } - } - - fn test_bus() -> (BusHandle, Addr>) { - let event_bus = - EventBus::::new(EventBusConfig { deduplicate: true }).start(); - let store = TestEventStore::default().start(); - let sequencer = Sequencer::new(&event_bus, store.recipient()).start(); - let bus = BusHandle::new(event_bus, sequencer, HlcFactory::new()).enable("test-keyshare"); - let history = bus.history(); - (bus, history) - } - - fn test_state() -> Persistable { - let store = InMemStore::new(false).start(); - let repo = Repository::::new(DataStore::from_in_mem(&store)); - repo.send(None) - } - - async fn start_actor() -> Result<( - Addr, - Addr>, - E3id, - )> { - let (bus, history) = test_bus(); - let actor = ThresholdKeyshare::new(ThresholdKeyshareParams { - bus, - cipher: Arc::new(Cipher::from_password("test-password").await?), - state: test_state(), - share_enc_preset: DEFAULT_BFV_PRESET, - }) - .start(); - - Ok((actor, history, E3id::new("42", 1))) - } - - async fn next_event( - history: &Addr>, - ) -> Result { - let mut result = history.send(TakeEvents::::new(1)).await?; - assert!(!result.timed_out, "timed out waiting for an event"); - Ok(result.events.pop().expect("expected one event")) - } - - async fn next_events( - history: &Addr>, - count: usize, - ) -> Result> { - let result = history - .send(TakeEvents::::new(count)) - .await?; - assert!(!result.timed_out, "timed out waiting for events"); - assert_eq!(result.events.len(), count, "expected {count} events"); - Ok(result.events) - } - - #[actix::test] - async fn encryption_key_collection_failure_preserves_telemetry_and_emits_e3_failed( - ) -> Result<()> { - let (actor, history, e3_id) = start_actor().await?; - let failure = EncryptionKeyCollectionFailed { - e3_id, - reason: "missing encryption keys".to_string(), - missing_parties: vec![2, 3], - }; - - actor.send(failure.clone()).await?; - - let mut events = next_events(&history, 2).await?; - let event = events.remove(0); - assert!(matches!( - event.into_data(), - InterfoldEventData::EncryptionKeyCollectionFailed(data) if data == failure - )); - - let event = events.remove(0); - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == failure.e3_id - && data.failed_at_stage == E3Stage::CommitteeFinalized - && data.reason == FailureReason::DKGTimeout - )); - - Ok(()) - } - - #[actix::test] - async fn threshold_share_collection_failure_preserves_telemetry_and_emits_e3_failed( - ) -> Result<()> { - let (actor, history, e3_id) = start_actor().await?; - let failure = ThresholdShareCollectionFailed { - e3_id, - reason: "missing threshold shares".to_string(), - missing_parties: vec![4, 5], - }; - - actor.send(failure.clone()).await?; - - let mut events = next_events(&history, 2).await?; - let event = events.remove(0); - assert!(matches!( - event.into_data(), - InterfoldEventData::ThresholdShareCollectionFailed(data) if data == failure - )); - - let event = events.remove(0); - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == failure.e3_id - && data.failed_at_stage == E3Stage::CommitteeFinalized - && data.reason == FailureReason::DKGTimeout - )); - - Ok(()) - } - - #[actix::test] - async fn decryption_key_shared_collection_failure_emits_e3_failed() -> Result<()> { - let (actor, history, e3_id) = start_actor().await?; - let failure = DecryptionKeySharedCollectionFailed { - e3_id, - reason: "missing decryption key shares".to_string(), - missing_parties: vec![6, 7], - }; - - actor.send(failure.clone()).await?; - - let event = next_event(&history).await?; - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == failure.e3_id - && data.failed_at_stage == E3Stage::CommitteeFinalized - && data.reason == FailureReason::DecryptionTimeout - )); - - Ok(()) - } -} diff --git a/crates/keyshare/src/domain/mod.rs b/crates/keyshare/src/domain.rs similarity index 62% rename from crates/keyshare/src/domain/mod.rs rename to crates/keyshare/src/domain.rs index 0106bb187a..d7bca7126a 100644 --- a/crates/keyshare/src/domain/mod.rs +++ b/crates/keyshare/src/domain.rs @@ -4,19 +4,26 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -//! Pure, synchronous domain logic for the threshold keyshare DKG flow. +//! Compatibility view of pure threshold-keyshare modules. //! //! Nothing in this module depends on actix, persistence, the event bus or -//! timers. The actors in [`crate::actors`] are thin message-passing shells that -//! delegate all decision-making to these unit-testable services. +//! timers. Implementation is physically co-located under `threshold_keyshare`. +#[path = "threshold_keyshare/generate_encryption_key.rs"] mod bfv_keygen; +#[path = "threshold_keyshare/derive_decryption_key.rs"] mod decryption_key_calculation; +#[path = "threshold_keyshare/collect_decryption_keys.rs"] mod decryption_key_shared_collection; +#[path = "threshold_keyshare/collect_encryption_keys.rs"] mod encryption_key_collection; +#[path = "threshold_keyshare/state.rs"] mod keyshare_state; +#[path = "threshold_keyshare/generate_shares.rs"] mod share_generation; +#[path = "threshold_keyshare/collect_threshold_shares.rs"] mod threshold_share_collection; +#[path = "threshold_keyshare/timeout_policy.rs"] pub(crate) mod timeout_policy; // Public (re-exported at the crate root): the persisted state machine and its diff --git a/crates/keyshare/src/threshold_keyshare/actor.rs b/crates/keyshare/src/threshold_keyshare/actor.rs new file mode 100644 index 0000000000..4d18c90669 --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/actor.rs @@ -0,0 +1,231 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::prelude::*; +use anyhow::{anyhow, bail, Context, Result}; +use e3_crypto::{Cipher, SensitiveBytes}; +use e3_data::Persistable; +use e3_events::{ + prelude::*, trap, BusHandle, CiphernodeSelected, CiphertextOutputPublished, + CommitteeMemberExpelled, ComputeRequest, ComputeResponse, ComputeResponseKind, CorrelationId, + DecryptionKeyShared, DecryptionShareProofSigned, DecryptionShareProofsPending, Die, + DkgProofSigned, DkgShareDecryptionProofRequest, E3Failed, E3RequestComplete, E3Stage, EType, + EncryptionKey, EncryptionKeyCollectionFailed, EncryptionKeyCreated, EncryptionKeyPending, + EventContext, FailureReason, InterfoldEvent, InterfoldEventData, KeyshareCreated, + PartyProofsToVerify, PartyShareDecryptionProofsToVerify, PkGenerationProofSigned, ProofType, + Sequenced, ShareDecryptionProofPending, ShareVerificationComplete, ShareVerificationDispatched, + SignedProofPayload, ThresholdShare, ThresholdShareCollectionFailed, ThresholdShareCreated, + ThresholdShareDecryptionProofRequest, ThresholdSharePending, TypedEvent, VerificationKind, +}; +use e3_fhe_params::create_deterministic_crp_from_default_seed; +use e3_fhe_params::BfvPreset; +use e3_trbfv::{ + calculate_decryption_key::CalculateDecryptionKeyResponse, + calculate_decryption_share::{ + CalculateDecryptionShareRequest, CalculateDecryptionShareResponse, + }, + gen_esi_sss::{GenEsiSssRequest, GenEsiSssResponse}, + gen_pk_share_and_sk_sss::{GenPkShareAndSkSssRequest, GenPkShareAndSkSssResponse}, + shares::SharedSecret, + TrBFVConfig, TrBFVRequest, TrBFVResponse, +}; +use e3_utils::utility_types::ArcBytes; +use e3_utils::{NotifySync, MAILBOX_LIMIT}; +use e3_zk_helpers::CiphernodesCommitteeSize; +use fhe_traits::Serialize; +use std::{ + collections::{BTreeSet, HashMap, HashSet}, + sync::Arc, +}; +use tracing::{error, info, trace, warn}; + +use crate::actors::decryption_key_shared_collector::{ + AllDecryptionKeySharesCollected, DecryptionKeySharedCollectionFailed, + DecryptionKeySharedCollector, ExpelPartyFromDecryptionKeySharedCollection, +}; +use crate::actors::encryption_key_collector::{ + AllEncryptionKeysCollected, EncryptionKeyCollector, ExpelPartyFromKeyCollection, +}; +use crate::actors::threshold_share_collector::{ + ExpelPartyFromShareCollection, ThresholdShareCollector, +}; +use crate::domain::timeout_policy::{resolve_timeout, DkgTimeoutPhase}; +use crate::domain::{ + build_decryption_key_plan, build_shares_generated_plan, generate_bfv_keypair, + AggregatingDecryptionKey, BfvKeypairMaterial, CollectingEncryptionKeysData, Decrypting, + DecryptionKeyPlan, GeneratingDecryptionProof, GeneratingThresholdShareData, KeyshareState, + ProofRequestData, ReadyForDecryption, ReceivedShareProofs, ThresholdKeyshareState, +}; + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[rtype(result = "()")] +pub struct GenPkShareAndSkSss(CiphernodeSelected); + +#[derive(Message, Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[rtype(result = "()")] +pub struct GenEsiSss { + pub ciphernode_selected: CiphernodeSelected, + pub e_sm_raw: SensitiveBytes, +} + +#[derive(Message)] +#[rtype(result = "()")] +pub struct AllThresholdSharesCollected { + /// Threshold shares sorted by ascending `party_id`. + shares: Vec>, + /// Proofs from each sender, ordered by party_id (parallel to shares). + share_proofs: Vec, +} + +impl AllThresholdSharesCollected { + pub fn new( + shares: HashMap>, + proofs: HashMap, + ) -> Self { + let mut entries: Vec<_> = shares.into_iter().collect(); + entries.sort_by_key(|(k, _)| *k); + let (party_ids, shares): (Vec<_>, Vec<_>) = entries.into_iter().unzip(); + let share_proofs = party_ids + .iter() + .map(|pid| { + proofs.get(pid).cloned().unwrap_or(ReceivedShareProofs { + signed_c2a_proof: None, + signed_c2b_proof: None, + signed_c3a_proofs: Vec::new(), + signed_c3b_proofs: Vec::new(), + }) + }) + .collect(); + Self { + shares, + share_proofs, + } + } +} + +pub struct ThresholdKeyshareParams { + pub bus: BusHandle, + pub cipher: Arc, + pub state: Persistable, + pub share_enc_preset: BfvPreset, +} + +/// Ephemeral bridge data for operations already represented by the persisted keyshare phase. +/// +/// Keeping these values behind one boundary makes their lifetime and recovery status explicit. A +/// recovery-safe replacement can move this bundle to a versioned workflow/outbox record without +/// growing the actor itself again. +#[derive(Default)] +struct PendingKeyshareWork { + /// Shares awaiting the C2/C3 verification result. + shares: Vec>, + /// C4 requests awaiting the threshold-decryption-key result. + share_decryption_data: Option<( + DkgShareDecryptionProofRequest, + Vec, + )>, + /// Peer C4 artifacts awaiting verification. + c4_verification_shares: Option>, + /// Own plaintext DKG shares awaiting the aggregation transition. + own_dkg_shares: Option<(SensitiveBytes, Vec)>, + /// C4 completed before the signed C1 artifact became available. + keyshare_publish: bool, +} + +pub struct ThresholdKeyshare { + bus: BusHandle, + cipher: Arc, + decryption_key_collector: Option>, + encryption_key_collector: Option>, + decryption_key_shared_collector: Option>, + state: Persistable, + share_enc_preset: BfvPreset, + pending: PendingKeyshareWork, +} + +impl ThresholdKeyshare { + pub fn new(params: ThresholdKeyshareParams) -> Self { + Self { + bus: params.bus, + cipher: params.cipher, + decryption_key_collector: None, + encryption_key_collector: None, + decryption_key_shared_collector: None, + state: params.state, + share_enc_preset: params.share_enc_preset, + pending: PendingKeyshareWork::default(), + } + } + + fn store_signed_pk_generation_proof( + &mut self, + ec: &EventContext, + signed: SignedProofPayload, + ) -> Result<()> { + self.state.try_mutate(ec, |mut s| { + match &mut s.state { + KeyshareState::AggregatingDecryptionKey(adk) => { + adk.signed_pk_generation_proof = Some(signed.clone()); + } + KeyshareState::ReadyForDecryption(rfd) => { + rfd.signed_pk_generation_proof = Some(signed.clone()); + } + KeyshareState::Decrypting(d) => { + d.signed_pk_generation_proof = Some(signed.clone()); + } + other => { + warn!( + "PkGenerationProofSigned in {:?} — C1 proof not stored (unexpected state)", + other.variant_name() + ); + } + } + Ok(s) + }) + } + + fn keyshare_created_fields( + state: &KeyshareState, + ) -> Option<(&ArcBytes, &Option)> { + use KeyshareState as K; + match state { + K::ReadyForDecryption(s) => Some((&s.pk_share, &s.signed_pk_generation_proof)), + K::Decrypting(s) => Some((&s.pk_share, &s.signed_pk_generation_proof)), + _ => None, + } + } + + fn try_finish_deferred_keyshare_publish(&mut self, ec: EventContext) -> Result<()> { + if !self.pending.keyshare_publish { + return Ok(()); + } + let state = self.state.try_get()?; + let Some((_, signed)) = Self::keyshare_created_fields(&state.state) else { + return Ok(()); + }; + if signed.is_none() { + return Ok(()); + } + self.pending.keyshare_publish = false; + self.publish_keyshare_created(ec) + } +} + +impl Actor for ThresholdKeyshare { + type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } +} + +#[path = "effects/mod.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +#[cfg(test)] +#[path = "tests.rs"] +mod tests; diff --git a/crates/keyshare/src/domain/decryption_key_shared_collection.rs b/crates/keyshare/src/threshold_keyshare/collect_decryption_keys.rs similarity index 100% rename from crates/keyshare/src/domain/decryption_key_shared_collection.rs rename to crates/keyshare/src/threshold_keyshare/collect_decryption_keys.rs diff --git a/crates/keyshare/src/domain/encryption_key_collection.rs b/crates/keyshare/src/threshold_keyshare/collect_encryption_keys.rs similarity index 100% rename from crates/keyshare/src/domain/encryption_key_collection.rs rename to crates/keyshare/src/threshold_keyshare/collect_encryption_keys.rs diff --git a/crates/keyshare/src/domain/threshold_share_collection.rs b/crates/keyshare/src/threshold_keyshare/collect_threshold_shares.rs similarity index 100% rename from crates/keyshare/src/domain/threshold_share_collection.rs rename to crates/keyshare/src/threshold_keyshare/collect_threshold_shares.rs diff --git a/crates/keyshare/src/actors/decryption_key_shared_collector.rs b/crates/keyshare/src/threshold_keyshare/decryption_key_collector.rs similarity index 100% rename from crates/keyshare/src/actors/decryption_key_shared_collector.rs rename to crates/keyshare/src/threshold_keyshare/decryption_key_collector.rs diff --git a/crates/keyshare/src/domain/decryption_key_calculation.rs b/crates/keyshare/src/threshold_keyshare/derive_decryption_key.rs similarity index 100% rename from crates/keyshare/src/domain/decryption_key_calculation.rs rename to crates/keyshare/src/threshold_keyshare/derive_decryption_key.rs diff --git a/crates/keyshare/src/threshold_keyshare/effects/calculate_decryption_key.rs b/crates/keyshare/src/threshold_keyshare/effects/calculate_decryption_key.rs new file mode 100644 index 0000000000..59e1302414 --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/effects/calculate_decryption_key.rs @@ -0,0 +1,194 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Decryption-key calculation and early C4 share handling. + +use super::*; + +impl ThresholdKeyshare { + /// After verification, decrypt shares from honest parties and compute the decryption key. + /// C4 proof generation is deferred to ProofRequestActor via DecryptionShareProofsPending. + pub(in crate::actors::threshold_keyshare) fn proceed_with_decryption_key_calculation( + &mut self, + dishonest_parties: Option>, + ec: EventContext, + ) -> Result<()> { + let state = self.state.try_get()?; + let e3_id = state.get_e3_id(); + let trbfv_config = state.get_trbfv_config(); + + // Get our BFV secret key from state, pending shares from the actor + let current: AggregatingDecryptionKey = state.clone().try_into()?; + let shares = std::mem::take(&mut self.pending.shares); + + let plan = build_decryption_key_plan( + &self.cipher, + self.share_enc_preset, + state.party_id, + state.threshold_m, + state.threshold_n, + trbfv_config, + ¤t, + shares, + dishonest_parties, + e3_id, + )?; + + match plan { + DecryptionKeyPlan::Insufficient => { + self.pending.shares.clear(); + self.bus.publish( + E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::InsufficientCommitteeMembers, + }, + ec, + )?; + } + DecryptionKeyPlan::Proceed { + calc_request, + sk_request, + esm_requests, + honest_party_ids, + } => { + // Publish CalculateDecryptionKey request before persisting (ordering preserved). + let event = ComputeRequest::trbfv( + TrBFVRequest::CalculateDecryptionKey(calc_request), + CorrelationId::new(), + e3_id.clone(), + ); + self.bus.publish(event, ec.clone())?; + + // Store honest parties and C4 data on the actor (transient coordination) + self.state.try_mutate(&ec, |mut s| { + s.honest_parties = Some(honest_party_ids.clone()); + Ok(s) + })?; + self.pending.share_decryption_data = Some((sk_request, esm_requests)); + } + } + + Ok(()) + } + + /// 5a. CalculateDecryptionKeyResponse — transition to ReadyForDecryption, + /// then publish DecryptionShareProofsPending so ProofRequestActor can + /// generate C4 proofs, sign them, and publish DecryptionKeyShared. + pub fn handle_calculate_decryption_key_response( + &mut self, + res: TypedEvent, + self_addr: Addr, + ) -> Result<()> { + let (res, ec) = res.into_components(); + let output: CalculateDecryptionKeyResponse = res + .try_into() + .context("Error extracting data from compute process")?; + + let (sk_poly_sum, es_poly_sum) = (output.sk_poly_sum, output.es_poly_sum); + + // Extract C4 data from the actor (stored by proceed_with_decryption_key_calculation) + let (sk_request, esm_requests) = self + .pending + .share_decryption_data + .take() + .ok_or_else(|| anyhow!("No pending share decryption data — CalculateDecryptionKey responded before proof requests were built"))?; + + // Take early shares from the actor before transitioning + let early_shares = self + .pending + .c4_verification_shares + .take() + .unwrap_or_default(); + + // Transition to ReadyForDecryption + self.state.try_mutate(&ec, |s| { + use KeyshareState as K; + info!("Try store decryption key"); + + let current: AggregatingDecryptionKey = s.clone().try_into()?; + + let next = K::ReadyForDecryption(ReadyForDecryption { + pk_share: current.pk_share, + sk_poly_sum, + es_poly_sum, + signed_pk_generation_proof: current.signed_pk_generation_proof, + signed_sk_share_computation_proof: current.signed_sk_share_computation_proof, + signed_e_sm_share_computation_proof: current.signed_e_sm_share_computation_proof, + signed_sk_share_encryption_proofs: current.signed_sk_share_encryption_proofs, + signed_e_sm_share_encryption_proofs: current.signed_e_sm_share_encryption_proofs, + }); + + s.new_state(next) + })?; + + // Publish DecryptionShareProofsPending to ProofRequestActor + let state = self.state.try_get()?; + let e3_id = state.get_e3_id(); + let party_id = state.party_id; + let node = state.address.clone(); + + info!( + "Publishing DecryptionShareProofsPending for E3 {} party {} (1 SK + {} ESM requests)", + e3_id, + party_id, + esm_requests.len() + ); + + self.bus.publish( + DecryptionShareProofsPending { + e3_id: e3_id.clone(), + party_id, + node, + sk_request, + esm_requests, + }, + ec.clone(), + )?; + + // Create collector and replay any early-arriving DecryptionKeyShared events + let state = self.state.try_get()?; + let my_party_id = state.party_id; + let honest = state.honest_parties.as_ref().cloned().unwrap_or_default(); + let expected: HashSet = honest + .iter() + .filter(|&&pid| pid != my_party_id) + .copied() + .collect(); + + if !expected.is_empty() { + let collector = self.ensure_decryption_key_shared_collector(self_addr)?; + for (_pid, share) in early_shares { + collector.do_send(TypedEvent::new(share, ec.clone())); + } + } + + Ok(()) + } + + /// Handle an external DecryptionKeyShared event while in AggregatingDecryptionKey state. + /// Store it for later processing when we transition to ReadyForDecryption. + pub(in crate::actors::threshold_keyshare) fn handle_early_decryption_key_share( + &mut self, + data: DecryptionKeyShared, + _ec: EventContext, + ) -> Result<()> { + let party_id = data.party_id; + let state = self.state.try_get()?; + if state.expelled_parties.contains(&party_id) { + info!( + "Dropping early DecryptionKeyShared from expelled party {}", + party_id + ); + return Ok(()); + } + info!( + "Storing early DecryptionKeyShared from party {} (state: AggregatingDecryptionKey)", + party_id + ); + self.pending + .c4_verification_shares + .get_or_insert_with(HashMap::new) + .insert(party_id, data); + Ok(()) + } +} diff --git a/crates/keyshare/src/threshold_keyshare/effects/coordinate_collectors.rs b/crates/keyshare/src/threshold_keyshare/effects/coordinate_collectors.rs new file mode 100644 index 0000000000..9d359a56ab --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/effects/coordinate_collectors.rs @@ -0,0 +1,246 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Collector lifecycle, expulsion, and early artifact routing. + +use super::*; + +impl ThresholdKeyshare { + pub fn ensure_collector( + &mut self, + self_addr: Addr, + ) -> Result> { + let Some(state) = self.state.get() else { + bail!("State not found on threshold keyshare. This should not happen."); + }; + + info!( + "Setting up key collector for addr: {} and {} nodes", + state.address, state.threshold_n + ); + let e3_id = state.e3_id.clone(); + let threshold_n = state.threshold_n; + let own_party_id = state.party_id; + let timeout = resolve_timeout( + DkgTimeoutPhase::ThresholdShareCollection, + state.dkg_started_at_unix_secs, + ); + info!( + e3_id = %e3_id, + timeout = ?timeout.duration, + "{}", + timeout.description + ); + let addr = self.decryption_key_collector.get_or_insert_with(|| { + ThresholdShareCollector::setup( + self_addr, + threshold_n, + own_party_id, + e3_id, + timeout.duration, + ) + }); + Ok(addr.clone()) + } + + pub fn ensure_encryption_key_collector( + &mut self, + self_addr: Addr, + ) -> Result> { + let Some(state) = self.state.get() else { + bail!("State not found on threshold keyshare. This should not happen."); + }; + + info!( + "Setting up encryption key collector for addr: {} and {} nodes", + state.address, state.threshold_n + ); + let e3_id = state.e3_id.clone(); + let threshold_n = state.threshold_n; + let timeout = resolve_timeout( + DkgTimeoutPhase::EncryptionKeyCollection, + state.dkg_started_at_unix_secs, + ); + info!( + e3_id = %e3_id, + timeout = ?timeout.duration, + "{}", + timeout.description + ); + let addr = self.encryption_key_collector.get_or_insert_with(|| { + EncryptionKeyCollector::setup(self_addr, threshold_n, e3_id, timeout.duration) + }); + Ok(addr.clone()) + } + + /// Create or return the DecryptionKeySharedCollector. + /// Uses honest_parties from persisted state. + pub fn ensure_decryption_key_shared_collector( + &mut self, + self_addr: Addr, + ) -> Result> { + let state = self.state.try_get()?; + let my_party_id = state.party_id; + + let honest = state + .honest_parties + .as_ref() + .ok_or_else(|| anyhow!("honest_parties not set when creating collector"))?; + + let expected: HashSet = honest + .iter() + .filter(|&&pid| pid != my_party_id) + .copied() + .collect(); + + let e3_id = state.e3_id.clone(); + let timeout = resolve_timeout( + DkgTimeoutPhase::DecryptionKeySharedCollection, + state.dkg_started_at_unix_secs, + ); + info!( + e3_id = %e3_id, + timeout = ?timeout.duration, + "{}", + timeout.description + ); + let addr = self.decryption_key_shared_collector.get_or_insert_with(|| { + DecryptionKeySharedCollector::setup(self_addr, expected, e3_id, timeout.duration) + }); + Ok(addr.clone()) + } + + pub(in crate::actors::threshold_keyshare) fn handle_committee_member_expelled( + &mut self, + data: CommitteeMemberExpelled, + ec: EventContext, + ) { + // Only process enriched events (party_id resolved by Sortition). + // Raw events from chain (party_id = None) are ignored here; + // Sortition will re-publish them with party_id set. + let Some(party_id) = data.party_id else { + return; + }; + + let node_addr = data.node.to_string(); + info!( + "CommitteeMemberExpelled received (enriched): node={}, party_id={}, e3_id={}, active_count_after={}", + node_addr, party_id, data.e3_id, data.active_count_after + ); + + // Record permanently so late-arriving data is rejected even if + // collectors haven't been created or have already completed. + // Also clean honest_parties set for the expelled party. + let _ = self.state.try_mutate(&ec, |mut s| { + s.expelled_parties.insert(party_id); + if let Some(ref mut honest) = s.honest_parties { + honest.remove(&party_id); + } + Ok(s) + }); + + // Clean transient coordination state for the expelled party + self.pending.shares.retain(|s| s.party_id != party_id); + + if let Some(ref mut pending_c4) = self.pending.c4_verification_shares { + pending_c4.remove(&party_id); + } + + if let Some(ref collector) = self.encryption_key_collector { + collector.do_send(ExpelPartyFromKeyCollection { + party_id, + ec: ec.clone(), + }); + } + + if let Some(ref collector) = self.decryption_key_collector { + collector.do_send(ExpelPartyFromShareCollection { + party_id, + ec: ec.clone(), + }); + } + + if let Some(ref collector) = self.decryption_key_shared_collector { + collector.do_send(ExpelPartyFromDecryptionKeySharedCollection { party_id, ec }); + } + } + + pub fn handle_threshold_share_created( + &mut self, + msg: TypedEvent, + self_addr: Addr, + ) -> Result<()> { + let state = self.state.try_get()?; + if !matches!( + state.state, + KeyshareState::CollectingEncryptionKeys(_) + | KeyshareState::GeneratingThresholdShare(_) + | KeyshareState::AggregatingDecryptionKey(_) + ) { + trace!( + e3_id = %state.e3_id, + state = state.variant_name(), + sender_party_id = msg.share.party_id, + "Ignoring ThresholdShareCreated outside share collection" + ); + return Ok(()); + } + + let my_party_id = state.party_id; + + // Filter: only process shares intended for this party + if msg.target_party_id != my_party_id { + return Ok(()); + } + + // Reject shares from expelled parties + if state.expelled_parties.contains(&msg.share.party_id) { + info!( + "Dropping ThresholdShareCreated from expelled party {} for us (party {})", + msg.share.party_id, my_party_id + ); + return Ok(()); + } + + info!( + "Received ThresholdShareCreated from party {} for us (party {}), forwarding to collector!", + msg.share.party_id, my_party_id + ); + let collector = self.ensure_collector(self_addr)?; + info!("got collector address!"); + collector.do_send(msg); + Ok(()) + } + + pub fn handle_encryption_key_created( + &mut self, + msg: TypedEvent, + self_addr: Addr, + ) -> Result<()> { + let state = self.state.try_get()?; + if !matches!( + state.state, + KeyshareState::Init | KeyshareState::CollectingEncryptionKeys(_) + ) { + trace!( + e3_id = %state.e3_id, + state = state.variant_name(), + sender_party_id = msg.key.party_id, + "Ignoring EncryptionKeyCreated outside key collection" + ); + return Ok(()); + } + + // Reject keys from expelled parties + if state.expelled_parties.contains(&msg.key.party_id) { + info!( + "Dropping EncryptionKeyCreated from expelled party {}", + msg.key.party_id + ); + return Ok(()); + } + info!("Received EncryptionKeyCreated forwarding to encryption key collector!"); + let collector = self.ensure_encryption_key_collector(self_addr)?; + collector.do_send(msg); + Ok(()) + } +} diff --git a/crates/keyshare/src/threshold_keyshare/effects/create_decryption_share.rs b/crates/keyshare/src/threshold_keyshare/effects/create_decryption_share.rs new file mode 100644 index 0000000000..25544eba8d --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/effects/create_decryption_share.rs @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Ciphertext handling, C6 share calculation, and restart redrive. + +use super::*; + +impl ThresholdKeyshare { + /// Handle the ciphertext output and begin local C6 decryption-share work. + pub fn handle_ciphertext_output_published( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + let ciphertext_output = msg.ciphertext_output; + + // If we are already in Decrypting (or beyond), this is a duplicate + // event — e.g. replayed from the EventStore during crash recovery. + // Re-issue the compute request idempotently (downstream aggregation + // deduplicates by party_id) and skip the state transition. + { + let state = self.state.try_get()?; + match &state.state { + KeyshareState::Decrypting(_) => { + info!( + e3_id = %state.e3_id, + "CiphertextOutputPublished received while already Decrypting — re-issuing decryption-share request" + ); + return self.issue_decryption_share_request(ec); + } + KeyshareState::GeneratingDecryptionProof(_) | KeyshareState::Completed => { + info!( + e3_id = %state.e3_id, + state = %state.variant_name(), + "CiphertextOutputPublished received after decryption already in progress — ignoring" + ); + return Ok(()); + } + KeyshareState::ReadyForDecryption(_) => { + // Normal path — proceed with transition below. + } + other => { + warn!( + e3_id = %state.e3_id, + state = %other.variant_name(), + "CiphertextOutputPublished received in unexpected state — cannot process decryption" + ); + return Ok(()); + } + } + } + + // Set state to decrypting, storing ciphertext for later C6 proof generation + self.state.try_mutate(&ec, |s| { + use KeyshareState as K; + + let current: ReadyForDecryption = s.clone().try_into()?; + + let next = K::Decrypting(Decrypting { + pk_share: current.pk_share, + sk_poly_sum: current.sk_poly_sum, + es_poly_sum: current.es_poly_sum, + ciphertext_output: ciphertext_output.clone(), + signed_pk_generation_proof: current.signed_pk_generation_proof, + signed_sk_share_computation_proof: current.signed_sk_share_computation_proof, + signed_e_sm_share_computation_proof: current.signed_e_sm_share_computation_proof, + signed_sk_share_encryption_proofs: current.signed_sk_share_encryption_proofs, + signed_e_sm_share_encryption_proofs: current.signed_e_sm_share_encryption_proofs, + }); + + s.new_state(next) + })?; + + let state = self.state.try_get()?; + let e3_id = state.get_e3_id(); + let decrypting: Decrypting = state.clone().try_into()?; + let trbfv_config = state.get_trbfv_config(); + let event = ComputeRequest::trbfv( + TrBFVRequest::CalculateDecryptionShare(CalculateDecryptionShareRequest { + name: format!("party_id({})", state.party_id), + ciphertexts: ciphertext_output, + sk_poly_sum: decrypting.sk_poly_sum, + es_poly_sum: decrypting.es_poly_sum, + trbfv_config, + }), + CorrelationId::new(), + e3_id.clone(), + ); + self.bus.publish(event, ec)?; // CalculateDecryptionShareRequest + Ok(()) + } + + /// (Re)issue the `CalculateDecryptionShare` compute request from the current + /// `Decrypting` state. Factored out of `handle_ciphertext_output_published` so the + /// boot-time resume path can re-drive the decryption-share computation idempotently + /// (the resulting `DecryptionshareCreated` is deduped by `party_id` at the aggregator). + pub(in crate::actors::threshold_keyshare) fn issue_decryption_share_request( + &self, + ec: EventContext, + ) -> Result<()> { + let state = self.state.try_get()?; + let e3_id = state.get_e3_id(); + let decrypting: Decrypting = state.clone().try_into()?; + let trbfv_config = state.get_trbfv_config(); + let event = ComputeRequest::trbfv( + TrBFVRequest::CalculateDecryptionShare(CalculateDecryptionShareRequest { + name: format!("party_id({})", state.party_id), + ciphertexts: decrypting.ciphertext_output, + sk_poly_sum: decrypting.sk_poly_sum, + es_poly_sum: decrypting.es_poly_sum, + trbfv_config, + }), + CorrelationId::new(), + e3_id.clone(), + ); + self.bus.publish(event, ec)?; + Ok(()) + } + + /// Re-drive this node's own in-flight DKG/decryption work after a crash or restart. + /// + /// Invoked when `EffectsEnabled` is broadcast at the end of boot sync, *after* this + /// actor has been hydrated from its persisted state. The dangerous, value-bearing + /// concern with re-driving is double-emission; this is safe here because every output + /// re-emitted below is deduplicated downstream by `party_id`: + /// * `KeyshareCreated` — `PublicKeyAggregation::add_keyshare` ignores a `party_id` + /// that already submitted (idempotent), and + /// * `DecryptionshareCreated` — threshold plaintext aggregation keys shares by + /// `party_id` (re-insert overwrites with the identical deterministic share). + /// + /// Only states where the local result is already determined are re-driven. Earlier + /// phases depend on peer gossip that cannot be reconstructed locally and are surfaced + /// (non-destructively) by `interfold node validate` instead of being force-re-driven. + pub(in crate::actors::threshold_keyshare) fn resume_in_flight_work( + &mut self, + ec: EventContext, + ) -> Result<()> { + let Some(state) = self.state.get() else { + return Ok(()); + }; + match &state.state { + // We have produced our public-key share but may have crashed before (or while) + // publishing KeyshareCreated. Re-publishing is idempotent at the aggregator, but + // ReadyForDecryption is entered *before* C4 honest-set verification authorizes the + // publish, so only re-drive when a prior authorized publish was recorded. An + // un-published ReadyForDecryption is a loose end surfaced by `interfold node validate`. + KeyshareState::ReadyForDecryption(_) if state.keyshare_published => { + info!( + e3_id = %state.e3_id, + "Resuming in-flight work: re-publishing KeyshareCreated" + ); + self.publish_keyshare_created(ec)?; + } + // The ciphertext to decrypt has arrived. Re-publish our keyshare (in case the + // crash happened before it propagated) and re-issue the decryption-share + // computation so a DecryptionshareCreated is (re)produced. + KeyshareState::Decrypting(_) => { + info!( + e3_id = %state.e3_id, + "Resuming in-flight work: re-publishing KeyshareCreated and re-issuing decryption-share request" + ); + self.publish_keyshare_created(ec.clone())?; + self.issue_decryption_share_request(ec)?; + } + other => { + trace!( + e3_id = %state.e3_id, + state = %other.variant_name(), + "No locally re-drivable work on resume; loose ends are surfaced by `interfold node validate`" + ); + } + } + Ok(()) + } + + /// CalculateDecryptionShareResponse — publish ShareDecryptionProofPending + /// so ProofRequestActor generates and signs C6 proofs. + pub fn handle_calculate_decryption_share_response( + &mut self, + res: TypedEvent, + ) -> Result<()> { + let (res, ec) = res.into_components(); + let msg: CalculateDecryptionShareResponse = res.try_into()?; + let state = self.state.try_get()?; + let e3_id = state.e3_id.clone(); + let decrypting: Decrypting = state.clone().try_into()?; + let d_share_poly = msg.d_share_poly; + + let aggregated_pk_bytes = state + .aggregated_pk + .clone() + .ok_or_else(|| anyhow!("Aggregated public key not available for C6 proof"))?; + + let threshold_preset = self + .share_enc_preset + .threshold_counterpart() + .ok_or_else(|| { + anyhow!( + "No threshold counterpart for preset {:?}", + self.share_enc_preset + ) + })?; + + info!("Publishing ShareDecryptionProofPending for C6 proof generation..."); + + let committee_size = CiphernodesCommitteeSize::from_threshold( + state.threshold_m as usize, + state.threshold_n as usize, + )?; + + // Publish pending event before transitioning state so a publish + // failure leaves us in Decrypting (retryable) rather than + // GeneratingDecryptionProof (no retry path). + self.bus.publish( + ShareDecryptionProofPending { + e3_id: e3_id.clone(), + party_id: state.party_id, + node: state.address.clone(), + decryption_share: d_share_poly.clone(), + proof_request: ThresholdShareDecryptionProofRequest { + ciphertext_bytes: decrypting.ciphertext_output, + aggregated_pk_bytes, + sk_poly_sum: decrypting.sk_poly_sum, + es_poly_sum: decrypting.es_poly_sum, + d_share_bytes: d_share_poly.clone(), + params_preset: threshold_preset, + committee_size, + }, + }, + ec.clone(), + )?; + + // Transition to GeneratingDecryptionProof state + self.state.try_mutate(&ec, |s| { + use KeyshareState as K; + s.new_state(K::GeneratingDecryptionProof(GeneratingDecryptionProof { + pk_share: decrypting.pk_share.clone(), + decryption_share: d_share_poly, + signed_pk_generation_proof: decrypting.signed_pk_generation_proof.clone(), + signed_sk_share_computation_proof: decrypting + .signed_sk_share_computation_proof + .clone(), + signed_e_sm_share_computation_proof: decrypting + .signed_e_sm_share_computation_proof + .clone(), + signed_sk_share_encryption_proofs: decrypting + .signed_sk_share_encryption_proofs + .clone(), + signed_e_sm_share_encryption_proofs: decrypting + .signed_e_sm_share_encryption_proofs + .clone(), + })) + })?; + Ok(()) + } + + pub fn handle_decryption_share_proof_signed( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (_msg, ec) = msg.into_components(); + + self.state.try_mutate(&ec, |s| { + use KeyshareState as K; + info!("Decryption share sending process is complete"); + s.new_state(K::Completed) + })?; + + Ok(()) + } +} diff --git a/crates/keyshare/src/threshold_keyshare/effects/generate_threshold_share.rs b/crates/keyshare/src/threshold_keyshare/effects/generate_threshold_share.rs new file mode 100644 index 0000000000..3ccbf11dd1 --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/effects/generate_threshold_share.rs @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! ESI generation and encrypted threshold-share publication. + +use super::*; + +impl ThresholdKeyshare { + /// Dispatch ESI Shamir-share generation. + pub fn handle_gen_esi_sss_requested(&self, msg: TypedEvent) -> Result<()> { + let (msg, ec) = msg.into_components(); + info!("GenEsiSss on ThresholdKeyshare"); + + let e_sm_raw = msg.e_sm_raw; + let CiphernodeSelected { e3_id, .. } = msg.ciphernode_selected; + + let state = self + .state + .get() + .ok_or(anyhow!("State not found on ThrehsoldKeyshare"))?; + + let trbfv_config = state.get_trbfv_config(); + + let event = ComputeRequest::trbfv( + TrBFVRequest::GenEsiSss(GenEsiSssRequest { + trbfv_config, + e_sm_raw, + }), + CorrelationId::new(), + e3_id, + ); + + self.bus.publish(event, ec)?; + Ok(()) + } + + /// 3a. GenEsiSss result + pub fn handle_gen_esi_sss_response(&mut self, res: TypedEvent) -> Result<()> { + let (res, ec) = res.into_components(); + let output: GenEsiSssResponse = res.try_into()?; + + let esi_sss = output.esi_sss; + + // First store esi_sss in GeneratingThresholdShareData + self.state.try_mutate(&ec, |s| { + info!("try_store_esi_sss"); + let current: GeneratingThresholdShareData = s.clone().try_into()?; + s.new_state(KeyshareState::GeneratingThresholdShare( + GeneratingThresholdShareData { + esi_sss: Some(esi_sss), + ..current + }, + )) + })?; + + info!("esi stored"); + + // Check if all data is ready, if so call handle_shares_generated BEFORE transitioning + let current: GeneratingThresholdShareData = self.state.try_get()?.try_into()?; + let ready = current.pk_share.is_some() + && current.sk_sss.is_some() + && current.esi_sss.is_some() + && current.e_sm_raw.is_some() + && current.proof_request_data.is_some(); + + if ready { + // Call handle_shares_generated while still in GeneratingThresholdShare state + self.handle_shares_generated(ec.clone())?; + + // Consume the own plaintext shares stashed transiently by handle_shares_generated. + let (own_sk_share_raw, own_esi_shares_raw) = + self.pending.own_dkg_shares.take().ok_or_else(|| { + anyhow!("pending_own_dkg_shares missing — handle_shares_generated did not run") + })?; + + // Now transition to AggregatingDecryptionKey with minimal state + self.state.try_mutate(&ec, |s| { + let current: GeneratingThresholdShareData = s.clone().try_into()?; + s.new_state(KeyshareState::AggregatingDecryptionKey( + AggregatingDecryptionKey { + pk_share: current.pk_share.expect("pk_share checked above"), + sk_bfv: current.sk_bfv, + own_sk_share_raw: own_sk_share_raw.clone(), + own_esi_shares_raw: own_esi_shares_raw.clone(), + signed_pk_generation_proof: None, + signed_sk_share_computation_proof: None, + signed_e_sm_share_computation_proof: None, + signed_sk_share_encryption_proofs: Vec::new(), + signed_e_sm_share_encryption_proofs: Vec::new(), + }, + )) + })?; + } + Ok(()) + } + + /// 4. SharesGenerated - Encrypt shares with BFV and publish + pub fn handle_shares_generated(&mut self, ec: EventContext) -> Result<()> { + let Some(ThresholdKeyshareState { + state: + KeyshareState::GeneratingThresholdShare(GeneratingThresholdShareData { + pk_share: Some(pk_share), + sk_sss: Some(sk_sss), + esi_sss: Some(esi_sss), + e_sm_raw: Some(e_sm_raw), + proof_request_data: Some(proof_request_data), + collected_encryption_keys, + .. + }), + party_id, + e3_id, + threshold_m, + threshold_n, + .. + }) = self.state.get() + else { + bail!("Invalid state - expected GeneratingThresholdShare with all data"); + }; + + // Decrypt our shares from local storage + let decrypted_sk_sss: SharedSecret = sk_sss.decrypt(&self.cipher)?; + let decrypted_esi_sss: Vec = esi_sss + .into_iter() + .map(|s| s.decrypt(&self.cipher)) + .collect::>()?; + + let plan = build_shares_generated_plan( + &self.cipher, + self.share_enc_preset, + party_id, + threshold_m, + threshold_n, + pk_share, + decrypted_sk_sss, + decrypted_esi_sss, + e_sm_raw, + proof_request_data, + &collected_encryption_keys, + )?; + + // Cache own plaintext share rows for the AggregatingDecryptionKey transition. + self.pending.own_dkg_shares = Some((plan.own_sk_share_raw, plan.own_esi_shares_raw)); + + let proof_aggregation_enabled = self + .state + .try_get() + .map(|s| s.proof_aggregation_enabled) + .unwrap_or(true); + + info!("Publishing ThresholdSharePending for E3 {}", e3_id); + + // Publish ThresholdSharePending - ProofRequestActor will generate proof, sign, and publish ThresholdShareCreated + self.bus.publish( + ThresholdSharePending { + e3_id, + full_share: Arc::new(plan.full_share), + proof_request: plan.proof_request, + sk_share_computation_request: plan.sk_share_computation_request, + e_sm_share_computation_request: plan.e_sm_share_computation_request, + sk_share_encryption_requests: plan.sk_share_encryption_requests, + e_sm_share_encryption_requests: plan.e_sm_share_encryption_requests, + recipient_party_ids: plan.recipient_party_ids, + proof_aggregation_enabled, + }, + ec, + )?; + + Ok(()) + } +} diff --git a/crates/keyshare/src/threshold_keyshare/effects/initialize_dkg.rs b/crates/keyshare/src/threshold_keyshare/effects/initialize_dkg.rs new file mode 100644 index 0000000000..12936fb590 --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/effects/initialize_dkg.rs @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Ciphernode selection, BFV key setup, and initial TrBFV requests. + +use super::*; + +impl ThresholdKeyshare { + /// Generate BFV keys for a selected ciphernode and publish `EncryptionKeyPending`. + pub fn handle_ciphernode_selected( + &mut self, + msg: TypedEvent, + address: Addr, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + let state = self.state.try_get()?; + if !matches!(state.state, KeyshareState::Init) { + info!( + e3_id = %state.e3_id, + state = state.variant_name(), + "Ignoring replayed CiphernodeSelected; keyshare already initialized" + ); + return Ok(()); + } + + info!("CiphernodeSelected received."); + // Ensure the collectors are created + let _ = self.ensure_collector(address.clone()); + let _ = self.ensure_encryption_key_collector(address.clone()); + + let BfvKeypairMaterial { + sk_bfv: sk_bfv_encrypted, + pk_bfv: pk_bfv_bytes, + } = generate_bfv_keypair(&self.share_enc_preset, &self.cipher)?; + + let e3_id = state.e3_id.clone(); + + self.state.try_mutate(&ec, |s| { + s.new_state(KeyshareState::CollectingEncryptionKeys( + CollectingEncryptionKeysData { + sk_bfv: sk_bfv_encrypted.clone(), + pk_bfv: pk_bfv_bytes.clone(), + ciphernode_selected: msg, + }, + )) + })?; + + let committee_size = CiphernodesCommitteeSize::from_threshold( + state.threshold_m as usize, + state.threshold_n as usize, + )?; + self.bus.publish( + EncryptionKeyPending { + e3_id, + key: Arc::new(EncryptionKey::new(state.party_id, pk_bfv_bytes)), + params_preset: self.share_enc_preset, + committee_size, + }, + ec, + )?; + + Ok(()) + } + + /// 1a. AllEncryptionKeysCollected - All BFV keys received, start share generation + pub fn handle_all_encryption_keys_collected( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + info!( + "AllEncryptionKeysCollected - {} keys received", + msg.keys.len() + ); + + let state = self.state.try_get()?; + let current: CollectingEncryptionKeysData = state.clone().try_into()?; + + // Filter out any keys from parties expelled after collection started + let filtered_keys: Vec<_> = if state.expelled_parties.is_empty() { + msg.keys + } else { + msg.keys + .into_iter() + .filter(|k| !state.expelled_parties.contains(&k.party_id)) + .collect() + }; + + self.state.try_mutate(&ec, |s| { + s.new_state(KeyshareState::GeneratingThresholdShare( + GeneratingThresholdShareData { + sk_sss: None, + pk_share: None, + esi_sss: None, + e_sm_raw: None, + sk_bfv: current.sk_bfv, + pk_bfv: current.pk_bfv, + collected_encryption_keys: filtered_keys, + ciphernode_selected: Some(current.ciphernode_selected.clone()), + proof_request_data: None, + }, + )) + })?; + + self.handle_gen_pk_share_and_sk_sss_requested(TypedEvent::new( + GenPkShareAndSkSss(current.ciphernode_selected), + ec, + ))?; + + Ok(()) + } + + /// 2. GenPkShareAndSkSss + pub fn handle_gen_pk_share_and_sk_sss_requested( + &self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + info!("GenPkShareAndSkSss on ThresholdKeyshare"); + let CiphernodeSelected { e3_id, .. } = msg.0; + let state = self + .state + .get() + .ok_or(anyhow!("State not found on ThrehsoldKeyshare"))?; + + let trbfv_config: TrBFVConfig = state.get_trbfv_config(); + + let crp = ArcBytes::from_bytes( + &create_deterministic_crp_from_default_seed(&trbfv_config.params()).to_bytes(), + ); + + let threshold_preset = self + .share_enc_preset + .threshold_counterpart() + .ok_or_else(|| anyhow!("No threshold counterpart for {:?}", self.share_enc_preset))?; + let defaults = threshold_preset + .search_defaults() + .ok_or_else(|| anyhow!("No search defaults for {:?}", threshold_preset))?; + + let event = ComputeRequest::trbfv( + TrBFVRequest::GenPkShareAndSkSss(GenPkShareAndSkSssRequest { + trbfv_config, + crp, + lambda: threshold_preset.lambda_config(), + num_ciphertexts: defaults.z as usize, + }), + CorrelationId::new(), + e3_id, + ); + + self.bus.publish(event, ec)?; + Ok(()) + } + + /// 2a. GenPkShareAndSkSss result + pub fn handle_gen_pk_share_and_sk_sss_response( + &mut self, + res: TypedEvent, + ) -> Result<()> { + let (res, ec) = res.into_components(); + + let output: GenPkShareAndSkSssResponse = res + .try_into() + .context("Error extracting data from compute process")?; + + let (pk_share, sk_sss, e_sm_raw) = ( + output.pk_share.clone(), + output.sk_sss, + output.e_sm_raw.clone(), + ); + + // Store proof request data for later use by ProofRequestActor + let proof_request_data = ProofRequestData { + pk0_share_raw: output.pk0_share_raw, + sk_raw: output.sk_raw, + eek_raw: output.eek_raw, + }; + + self.state.try_mutate(&ec, |s| { + info!("try_store_pk_share_and_sk_sss"); + let current: GeneratingThresholdShareData = s.clone().try_into()?; + s.new_state(KeyshareState::GeneratingThresholdShare( + GeneratingThresholdShareData { + pk_share: Some(pk_share), + sk_sss: Some(sk_sss), + e_sm_raw: Some(e_sm_raw.clone()), + proof_request_data: Some(proof_request_data), + ..current + }, + )) + })?; + + // Fire gen_esi_sss with the e_sm_raw + let current_state: GeneratingThresholdShareData = self.state.try_get()?.try_into()?; + if let Some(ciphernode_selected) = current_state.ciphernode_selected { + self.handle_gen_esi_sss_requested(TypedEvent::new( + GenEsiSss { + ciphernode_selected, + e_sm_raw: current_state + .e_sm_raw + .expect("e_sm_raw should be set at this point"), + }, + ec.clone(), + ))?; + } + + Ok(()) + } +} diff --git a/crates/keyshare/src/threshold_keyshare/effects/mod.rs b/crates/keyshare/src/threshold_keyshare/effects/mod.rs new file mode 100644 index 0000000000..ae49d5bbe5 --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/effects/mod.rs @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Effect execution for threshold key generation and decryption. + +use super::*; + +mod calculate_decryption_key; +mod coordinate_collectors; +mod create_decryption_share; +mod generate_threshold_share; +mod initialize_dkg; +mod route_events; +mod track_proofs; +mod verify_decryption_key; +mod verify_threshold_shares; diff --git a/crates/keyshare/src/threshold_keyshare/effects/route_events.rs b/crates/keyshare/src/threshold_keyshare/effects/route_events.rs new file mode 100644 index 0000000000..34a7a383cb --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/effects/route_events.rs @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Route the per-E3 event envelope to typed keyshare handlers. + +use super::*; + +impl Handler for ThresholdKeyshare { + type Result = (); + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + match msg { + InterfoldEventData::CiphernodeSelected(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::CiphertextOutputPublished(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::PublicKeyAggregated(data) => { + let pk = ArcBytes::from_bytes(&data.pubkey); + let _ = self.state.try_mutate(&ec, |mut s| { + s.aggregated_pk = Some(pk); + Ok(s) + }); + } + InterfoldEventData::ThresholdShareCreated(data) => { + let _ = + self.handle_threshold_share_created(TypedEvent::new(data, ec), ctx.address()); + } + InterfoldEventData::EncryptionKeyCreated(data) => { + let _ = + self.handle_encryption_key_created(TypedEvent::new(data, ec), ctx.address()); + } + InterfoldEventData::PkGenerationProofSigned(data) => { + let _ = self.handle_pk_generation_proof_signed(TypedEvent::new(data, ec)); + } + InterfoldEventData::DkgProofSigned(data) => { + let _ = self.handle_share_computation_proof_signed(TypedEvent::new(data, ec)); + } + InterfoldEventData::E3RequestComplete(data) => self.notify_sync(ctx, data), + InterfoldEventData::E3Failed(data) => { + warn!( + "E3 failed: {:?}. Shutting down ThresholdKeyshare for e3_id={}", + data.reason, data.e3_id + ); + self.notify_sync(ctx, E3RequestComplete { e3_id: data.e3_id }); + } + InterfoldEventData::E3StageChanged(data) => { + use e3_events::E3Stage; + match &data.new_stage { + E3Stage::Complete | E3Stage::Failed => { + info!("E3 reached terminal stage {:?}. Shutting down ThresholdKeyshare for e3_id={}", data.new_stage, data.e3_id); + self.notify_sync(ctx, E3RequestComplete { e3_id: data.e3_id }); + } + _ => { + trace!( + "E3 stage changed to {:?} for e3_id={}", + data.new_stage, + data.e3_id + ); + } + } + } + InterfoldEventData::DecryptionKeyShared(data) => { + if data.external { + // Route based on current state + if let Some(state) = self.state.get() { + if state.expelled_parties.contains(&data.party_id) { + info!( + "Dropping DecryptionKeyShared from expelled party {}", + data.party_id + ); + return; + } + let result = match &state.state { + KeyshareState::AggregatingDecryptionKey(_) => { + self.handle_early_decryption_key_share(data, ec) + } + KeyshareState::ReadyForDecryption(_) => { + // Delegate to the collector actor + if let Some(ref collector) = self.decryption_key_shared_collector { + collector.do_send(TypedEvent::new(data, ec)); + Ok(()) + } else { + warn!( + "DecryptionKeyShared from party {} dropped — no collector (sole honest party)", + data.party_id + ); + Ok(()) + } + } + other => { + trace!( + "DecryptionKeyShared from party {} in unexpected state {:?}, ignoring", + data.party_id, + other.variant_name() + ); + Ok(()) + } + }; + if let Err(err) = result { + error!("Failed to handle DecryptionKeyShared: {err}"); + } + } + } else { + // Own DecryptionKeyShared published by ProofRequestActor. + // A3 fast-path: if no other honest parties, publish KeyshareCreated directly. + if let Some(state) = self.state.get() { + if data.party_id == state.party_id { + if let KeyshareState::ReadyForDecryption(_) = state.state { + let others = state + .honest_parties + .as_ref() + .map(|h| h.iter().filter(|&&pid| pid != state.party_id).count()) + .unwrap_or(0); + if others == 0 { + info!( + "No other honest parties for E3 {} — publishing KeyshareCreated directly", + data.e3_id + ); + if let Err(err) = self.publish_keyshare_created(ec) { + error!("Failed to publish KeyshareCreated: {err}"); + } + } + } + } + } + } + } + InterfoldEventData::DecryptionShareProofSigned(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ShareVerificationComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ComputeResponse(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::CommitteeMemberExpelled(data) => { + self.handle_committee_member_expelled(data, ec); + } + InterfoldEventData::EffectsEnabled(_) => { + // Broadcast once at the end of boot sync. Re-drive any of this node's own + // in-flight work that a crash may have interrupted (idempotent downstream). + if let Err(err) = self.resume_in_flight_work(ec) { + warn!("resume_in_flight_work failed: {err}"); + } + } + _ => (), + } + } +} diff --git a/crates/keyshare/src/threshold_keyshare/effects/track_proofs.rs b/crates/keyshare/src/threshold_keyshare/effects/track_proofs.rs new file mode 100644 index 0000000000..5426b6d64a --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/effects/track_proofs.rs @@ -0,0 +1,110 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Signed-proof state updates and compute-result routing. + +use super::*; + +impl ThresholdKeyshare { + /// Store the signed C1 proof in workflow state. + pub fn handle_pk_generation_proof_signed( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + let state = self.state.try_get()?; + + // Only accept proof for our own party + if msg.party_id != state.party_id { + return Ok(()); + } + + info!( + "Received PkGenerationProofSigned for party {} E3 {}", + msg.party_id, msg.e3_id + ); + + self.store_signed_pk_generation_proof(&ec, msg.signed_proof)?; + self.try_finish_deferred_keyshare_publish(ec)?; + + Ok(()) + } + + /// Handle DkgProofSigned - stores the signed proof in state based on proof type (C2a, C2b, C3a or C3b) + pub fn handle_share_computation_proof_signed( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + let state = self.state.try_get()?; + + if msg.party_id != state.party_id { + return Ok(()); + } + + let proof_type = msg.signed_proof.payload.proof_type; + info!( + "Received DkgProofSigned ({:?}) for party {} E3 {}", + proof_type, msg.party_id, msg.e3_id + ); + + self.state.try_mutate(&ec, |s| { + let current: AggregatingDecryptionKey = s.clone().try_into()?; + let updated = match proof_type { + ProofType::C2aSkShareComputation => AggregatingDecryptionKey { + signed_sk_share_computation_proof: Some(msg.signed_proof), + ..current + }, + ProofType::C2bESmShareComputation => AggregatingDecryptionKey { + signed_e_sm_share_computation_proof: Some(msg.signed_proof), + ..current + }, + ProofType::C3aSkShareEncryption => { + let mut updated = current; + updated + .signed_sk_share_encryption_proofs + .push(msg.signed_proof); + updated + } + ProofType::C3bESmShareEncryption => { + let mut updated = current; + updated + .signed_e_sm_share_encryption_proofs + .push(msg.signed_proof); + updated + } + other => { + warn!("Unexpected proof type {:?} in DkgProofSigned", other); + current + } + }; + s.new_state(KeyshareState::AggregatingDecryptionKey(updated)) + })?; + + Ok(()) + } + + pub fn handle_compute_response( + &mut self, + msg: TypedEvent, + self_addr: Addr, + ) -> Result<()> { + match &msg.response { + ComputeResponseKind::TrBFV(trbfv) => match trbfv { + TrBFVResponse::GenEsiSss(_) => self.handle_gen_esi_sss_response(msg), + TrBFVResponse::GenPkShareAndSkSss(_) => { + self.handle_gen_pk_share_and_sk_sss_response(msg) + } + TrBFVResponse::CalculateDecryptionKey(_) => { + self.handle_calculate_decryption_key_response(msg, self_addr) + } + TrBFVResponse::CalculateDecryptionShare(_) => { + self.handle_calculate_decryption_share_response(msg) + } + _ => Ok(()), + }, + // ZK responses: proofs and verification are handled by + // ProofRequestActor and ShareVerificationActor respectively. + ComputeResponseKind::Zk(_) => Ok(()), + } + } +} diff --git a/crates/keyshare/src/threshold_keyshare/effects/verify_decryption_key.rs b/crates/keyshare/src/threshold_keyshare/effects/verify_decryption_key.rs new file mode 100644 index 0000000000..6f246add41 --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/effects/verify_decryption_key.rs @@ -0,0 +1,174 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! C4 verification and keyshare publication. + +use super::*; + +impl ThresholdKeyshare { + /// Dispatch C4 verification for all collected DecryptionKeyShared events. + /// Shares are provided by the DecryptionKeySharedCollector. + pub(in crate::actors::threshold_keyshare) fn dispatch_c4_verification( + &mut self, + collected_shares: HashMap, + ec: EventContext, + ) -> Result<()> { + let state = self.state.try_get()?; + let e3_id = state.get_e3_id(); + let ready: ReadyForDecryption = state.clone().try_into()?; + + info!( + "All DecryptionKeyShared collected for E3 {} ({} shares)", + e3_id, + collected_shares.len() + ); + + // Validate ESM proof count — each party must provide exactly + // one C4b proof per smudging noise index. + let expected_esm = ready.es_poly_sum.len(); + let mut c4_count_dishonest: HashSet = HashSet::new(); + let party_proofs: Vec = collected_shares + .iter() + .filter_map(|(&party_id, share)| { + if share.signed_e_sm_decryption_proofs.len() != expected_esm { + warn!( + "Party {} has wrong ESM proof count ({} vs expected {}) for E3 {} — treating as dishonest", + party_id, + share.signed_e_sm_decryption_proofs.len(), + expected_esm, + e3_id + ); + c4_count_dishonest.insert(party_id); + None + } else { + Some(PartyShareDecryptionProofsToVerify { + sender_party_id: party_id, + signed_sk_decryption_proof: share.signed_sk_decryption_proof.clone(), + signed_e_sm_decryption_proofs: share.signed_e_sm_decryption_proofs.clone(), + }) + } + }) + .collect(); + + // Evict pre-dishonest parties (wrong ESM count) from honest set + if !c4_count_dishonest.is_empty() { + self.state.try_mutate(&ec, |mut s| { + if let Some(ref mut honest) = s.honest_parties { + honest.retain(|pid| !c4_count_dishonest.contains(pid)); + } + Ok(s) + })?; + } + + if party_proofs.is_empty() { + // Check threshold viability after removing pre-dishonest parties + let state = self.state.try_get()?; + let threshold = state.threshold_m; + let honest_count = state + .honest_parties + .as_ref() + .map(|h| h.len() as u64) + .unwrap_or(0); + + if honest_count <= threshold { + warn!( + "Too few honest parties after C4 pre-filtering for E3 {} ({} honest, need at least {})", + e3_id, honest_count, threshold + 1 + ); + self.bus.publish( + E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::InsufficientCommitteeMembers, + }, + ec, + )?; + return Ok(()); + } + + info!("No C4 proofs to verify — publishing KeyshareCreated directly"); + return self.publish_keyshare_created(ec); + } + + let pre_dishonest: BTreeSet = c4_count_dishonest.into_iter().collect(); + + info!( + "Dispatching C4 share verification for E3 {} ({} parties, {} pre-dishonest)", + e3_id, + party_proofs.len(), + pre_dishonest.len() + ); + + let committee_size = CiphernodesCommitteeSize::from_threshold( + state.threshold_m as usize, + state.threshold_n as usize, + )?; + self.bus.publish( + ShareVerificationDispatched { + e3_id: e3_id.clone(), + kind: VerificationKind::DecryptionProofs, + share_proofs: Vec::new(), + decryption_proofs: party_proofs, + pre_dishonest, + params_preset: self.share_enc_preset, + committee_size, + }, + ec, + )?; + Ok(()) + } + + /// Publish KeyshareCreated (Exchange #4) with pk_share and signed C1 proof. + pub(in crate::actors::threshold_keyshare) fn publish_keyshare_created( + &mut self, + ec: EventContext, + ) -> Result<()> { + let state = self.state.try_get()?; + let e3_id = state.get_e3_id(); + let address = state.get_address().to_owned(); + let party_id = state.get_party_id(); + let Some((pk_share, signed_pk_generation_proof)) = + Self::keyshare_created_fields(&state.state) + else { + warn!( + "Deferring KeyshareCreated for party {} E3 {} — not in ReadyForDecryption/Decrypting ({})", + party_id, + e3_id, + state.state.variant_name() + ); + self.pending.keyshare_publish = true; + return Ok(()); + }; + + if signed_pk_generation_proof.is_none() { + warn!( + "Deferring KeyshareCreated for party {} E3 {} — C1 proof not stored yet (PkGenerationProofSigned race)", + party_id, e3_id + ); + self.pending.keyshare_publish = true; + return Ok(()); + } + + info!("Publishing Exchange #4 (KeyshareCreated) for E3 {}", e3_id); + + self.bus.publish( + KeyshareCreated { + pubkey: pk_share.clone(), + e3_id: e3_id.clone(), + node: address, + party_id, + signed_pk_generation_proof: signed_pk_generation_proof.clone(), + }, + ec.clone(), + )?; + + // Record that publishing was authorized and has occurred, so resume-after-crash + // may safely re-publish (idempotent at the aggregator) without ever emitting a + // keyshare for a state that had not yet passed C4 honest-set filtering. + self.state.try_mutate(&ec, |mut s| { + s.keyshare_published = true; + Ok(s) + })?; + + Ok(()) + } +} diff --git a/crates/keyshare/src/threshold_keyshare/effects/verify_threshold_shares.rs b/crates/keyshare/src/threshold_keyshare/effects/verify_threshold_shares.rs new file mode 100644 index 0000000000..4a4a4dd43b --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/effects/verify_threshold_shares.rs @@ -0,0 +1,284 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! C2/C3 collection, verification dispatch, and result application. + +use super::*; + +impl ThresholdKeyshare { + /// Verify the collected C2/C3 proofs before decryption-key aggregation. + pub fn handle_all_threshold_shares_collected( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + info!("AllThresholdSharesCollected"); + let state = self.state.try_get()?; + let e3_id = state.get_e3_id(); + let own_party_id = state.party_id; + + // Filter out expelled parties before any processing. The collector may + // have accepted shares before the expulsion arrived, so we scrub here. + let expelled = &state.expelled_parties; + let (shares, share_proofs): (Vec<_>, Vec<_>) = if expelled.is_empty() { + (msg.shares, msg.share_proofs) + } else { + warn!( + "Filtering {} expelled parties from AllThresholdSharesCollected for E3 {}: {:?}", + expelled.len(), + e3_id, + expelled + ); + msg.shares + .into_iter() + .zip(msg.share_proofs) + .filter(|(s, _)| !expelled.contains(&s.party_id)) + .unzip() + }; + + // Expected proof counts come from local cached own shares (trusted source); the + // collector excludes self from `shares`, so we cannot read them from there. + let current: AggregatingDecryptionKey = state.clone().try_into()?; + let own_sk_rows: Vec> = + bincode::deserialize(¤t.own_sk_share_raw.access_raw(&self.cipher)?) + .context("Failed to deserialize own_sk_share_raw")?; + let expected_c3a = own_sk_rows.len(); + let expected_num_esi = current.own_esi_shares_raw.len(); + let mut expected_c3b: usize = 0; + for esi_raw in current.own_esi_shares_raw.iter() { + let rows: Vec> = bincode::deserialize(&esi_raw.access_raw(&self.cipher)?) + .context("Failed to deserialize own esi share")?; + expected_c3b += rows.len(); + } + + // Build verification requests for other parties' proofs + let mut party_proofs_to_verify: Vec = Vec::new(); + let mut no_proof_parties: HashSet = HashSet::new(); + let mut incomplete_proof_parties: HashSet = HashSet::new(); + for (share, proofs) in shares.iter().zip(share_proofs.iter()) { + if share.party_id == own_party_id { + continue; + } + + let has_any_proof = proofs.signed_c2a_proof.is_some() + || proofs.signed_c2b_proof.is_some() + || !proofs.signed_c3a_proofs.is_empty() + || !proofs.signed_c3b_proofs.is_empty(); + + if !has_any_proof { + no_proof_parties.insert(share.party_id); + continue; + } + + // Validate proof set completeness against trusted expected counts. + // A malicious sender could omit proofs that would fail verification, + // so we must check that all expected proofs are present. + let is_complete = proofs.signed_c2a_proof.is_some() + && proofs.signed_c2b_proof.is_some() + && proofs.signed_c3a_proofs.len() == expected_c3a + && proofs.signed_c3b_proofs.len() == expected_c3b + && share.esi_sss.len() == expected_num_esi; + + if !is_complete { + warn!( + "Party {} has incomplete proof set (c2a={}, c2b={}, c3a={}/{}, c3b={}/{}, esi={}/{}), treating as dishonest", + share.party_id, + proofs.signed_c2a_proof.is_some(), + proofs.signed_c2b_proof.is_some(), + proofs.signed_c3a_proofs.len(), expected_c3a, + proofs.signed_c3b_proofs.len(), expected_c3b, + share.esi_sss.len(), expected_num_esi, + ); + incomplete_proof_parties.insert(share.party_id); + continue; + } + + // Complete proof set — collect for verification + let mut signed_proofs = Vec::new(); + // SAFETY: is_complete guarantees c2a and c2b are Some + signed_proofs.push(proofs.signed_c2a_proof.clone().unwrap()); + signed_proofs.push(proofs.signed_c2b_proof.clone().unwrap()); + signed_proofs.extend(proofs.signed_c3a_proofs.iter().cloned()); + signed_proofs.extend(proofs.signed_c3b_proofs.iter().cloned()); + + party_proofs_to_verify.push(PartyProofsToVerify { + sender_party_id: share.party_id, + signed_proofs, + }); + } + + // Store shares on the actor for use after verification completes (keep Arc to avoid deep clone) + self.pending.shares = shares.to_vec(); + + // Merge no-proof and incomplete-proof parties — both are dishonest + let mut pre_dishonest: BTreeSet = BTreeSet::new(); + pre_dishonest.extend(incomplete_proof_parties); + pre_dishonest.extend(no_proof_parties); + if !pre_dishonest.is_empty() { + warn!( + "{} parties have missing/incomplete C2/C3 proofs for E3 {} — marking as pre-dishonest: {:?}", + pre_dishonest.len(), + e3_id, + pre_dishonest + ); + } + + if party_proofs_to_verify.is_empty() { + // All non-self parties are dishonest (missing or incomplete proofs), none to verify + let threshold = state.threshold_m; + let total = state.threshold_n; + let dishonest_count = (pre_dishonest.len() as u64).min(total); + let honest_count = total - dishonest_count; + + if honest_count <= threshold { + warn!( + "Too few honest parties for E3 {} ({} honest, need at least {}) after C2/C3 pre-dishonest filtering — cannot proceed", + e3_id, honest_count, threshold + 1 + ); + self.pending.shares.clear(); + self.bus.publish( + E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::InsufficientCommitteeMembers, + }, + ec, + )?; + return Ok(()); + } + + let dishonest_set: HashSet = pre_dishonest.into_iter().collect(); + return self.proceed_with_decryption_key_calculation(Some(dishonest_set), ec); + } + + info!( + "Dispatching C2/C3 share verification for E3 {} ({} parties, {} pre-dishonest)", + e3_id, + party_proofs_to_verify.len(), + pre_dishonest.len() + ); + + let committee_size = CiphernodesCommitteeSize::from_threshold( + state.threshold_m as usize, + state.threshold_n as usize, + )?; + self.bus.publish( + ShareVerificationDispatched { + e3_id: e3_id.clone(), + kind: VerificationKind::ShareProofs, + share_proofs: party_proofs_to_verify, + decryption_proofs: Vec::new(), + pre_dishonest, + params_preset: self.share_enc_preset, + committee_size, + }, + ec, + )?; + Ok(()) + } + + /// Handle ShareVerificationComplete from ShareVerificationActor. + /// Dispatched for both C2/C3 and C4 verification. + pub fn handle_share_verification_complete( + &mut self, + msg: TypedEvent, + ) -> Result<()> { + let (msg, ec) = msg.into_components(); + let state = self.state.try_get()?; + let e3_id = state.get_e3_id(); + + match msg.kind { + VerificationKind::ShareProofs => { + // C2/C3 verification complete + if msg.dishonest_parties.is_empty() { + info!( + "All parties passed C2/C3 verification for E3 {} — proceeding", + e3_id + ); + self.proceed_with_decryption_key_calculation(None, ec) + } else { + let threshold = state.threshold_m; + let total = state.threshold_n; + let dishonest_count = (msg.dishonest_parties.len() as u64).min(total); + let honest_count = total - dishonest_count; + + if honest_count <= threshold { + warn!( + "Too few honest parties for E3 {} ({} honest, need at least {}) — cannot proceed", + e3_id, honest_count, threshold + 1 + ); + // Clear pending shares + self.pending.shares.clear(); + self.bus.publish( + E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::InsufficientCommitteeMembers, + }, + ec, + )?; + return Ok(()); + } + + let dishonest_set: HashSet = msg.dishonest_parties.into_iter().collect(); + info!( + "Proceeding with {} honest parties for E3 {} ({} dishonest excluded)", + honest_count, + e3_id, + dishonest_set.len() + ); + self.proceed_with_decryption_key_calculation(Some(dishonest_set), ec) + } + } + VerificationKind::DecryptionProofs => { + // C4 verification complete — update honest set and publish KeyshareCreated + if !msg.dishonest_parties.is_empty() { + self.state.try_mutate(&ec, |mut s| { + if let Some(ref mut honest) = s.honest_parties { + honest.retain(|pid| !msg.dishonest_parties.contains(pid)); + } + Ok(s) + })?; + + let state = self.state.try_get()?; + let threshold = state.threshold_m; + let honest_count = state + .honest_parties + .as_ref() + .map(|h| h.len() as u64) + .unwrap_or(0); + + if honest_count <= threshold { + warn!( + "Too few honest parties after C4 for E3 {} ({} honest, need at least {})", + e3_id, honest_count, threshold + 1 + ); + self.bus.publish( + E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::InsufficientCommitteeMembers, + }, + ec, + )?; + return Ok(()); + } + + info!( + "Updated honest set after C4 for E3 {}: {} honest ({} removed)", + e3_id, + honest_count, + msg.dishonest_parties.len() + ); + } else { + info!( + "All parties passed C4 verification for E3 {} — publishing KeyshareCreated", + e3_id + ); + } + + self.publish_keyshare_created(ec) + } + _ => Ok(()), + } + } +} diff --git a/crates/keyshare/src/actors/encryption_key_collector.rs b/crates/keyshare/src/threshold_keyshare/encryption_key_collector.rs similarity index 100% rename from crates/keyshare/src/actors/encryption_key_collector.rs rename to crates/keyshare/src/threshold_keyshare/encryption_key_collector.rs diff --git a/crates/keyshare/src/domain/bfv_keygen.rs b/crates/keyshare/src/threshold_keyshare/generate_encryption_key.rs similarity index 100% rename from crates/keyshare/src/domain/bfv_keygen.rs rename to crates/keyshare/src/threshold_keyshare/generate_encryption_key.rs diff --git a/crates/keyshare/src/domain/share_generation.rs b/crates/keyshare/src/threshold_keyshare/generate_shares.rs similarity index 100% rename from crates/keyshare/src/domain/share_generation.rs rename to crates/keyshare/src/threshold_keyshare/generate_shares.rs diff --git a/crates/keyshare/src/threshold_keyshare/handlers.rs b/crates/keyshare/src/threshold_keyshare/handlers.rs new file mode 100644 index 0000000000..5a5ebd20d6 --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/handlers.rs @@ -0,0 +1,244 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Typed Actix handlers, failure mapping, and actor cleanup. + +use super::*; + +impl Handler> for ThresholdKeyshare { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_decryption_share_proof_signed(msg), + ) + } +} + +impl Handler> for ThresholdKeyshare { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_compute_response(msg, ctx.address()), + ) + } +} + +impl Handler> for ThresholdKeyshare { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_ciphernode_selected(msg, ctx.address()), + ) + } +} + +impl Handler> for ThresholdKeyshare { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_all_encryption_keys_collected(msg), + ) + } +} + +impl Handler> for ThresholdKeyshare { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_share_verification_complete(msg), + ) + } +} + +impl Handler> for ThresholdKeyshare { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_all_threshold_shares_collected(msg), + ) + } +} + +impl Handler> for ThresholdKeyshare { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_ciphertext_output_published(msg), + ) + } +} + +impl Handler for ThresholdKeyshare { + type Result = (); + fn handle( + &mut self, + msg: EncryptionKeyCollectionFailed, + ctx: &mut Self::Context, + ) -> Self::Result { + trap(EType::KeyGeneration, &self.bus.clone(), || { + warn!( + e3_id = %msg.e3_id, + missing_parties = ?msg.missing_parties, + "Encryption key collection failed: {}", + msg.reason + ); + + // Clear the collector reference since it's stopped + self.encryption_key_collector = None; + + // Publish failure event to event bus for sync tracking + self.bus.publish_without_context(msg.clone())?; + + self.bus.publish_without_context(E3Failed { + e3_id: msg.e3_id, + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGTimeout, + })?; + + // Stop this actor since we can't proceed without all encryption keys + ctx.stop(); + Ok(()) + }) + } +} + +impl Handler for ThresholdKeyshare { + type Result = (); + fn handle( + &mut self, + msg: ThresholdShareCollectionFailed, + ctx: &mut Self::Context, + ) -> Self::Result { + trap(EType::KeyGeneration, &self.bus.clone(), || { + warn!( + e3_id = %msg.e3_id, + missing_parties = ?msg.missing_parties, + "Threshold share collection failed: {}", + msg.reason + ); + + // Clear the collector reference since it's stopped + self.decryption_key_collector = None; + + // Publish failure event to event bus for sync tracking + self.bus.publish_without_context(msg.clone())?; + + self.bus.publish_without_context(E3Failed { + e3_id: msg.e3_id, + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGTimeout, + })?; + + ctx.stop(); + Ok(()) + }) + } +} + +impl Handler> for ThresholdKeyshare { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap( + EType::KeyGeneration, + &self.bus.with_ec(msg.get_ctx()), + || { + let (msg, ec) = msg.into_components(); + self.decryption_key_shared_collector = None; + self.dispatch_c4_verification(msg.shares, ec) + }, + ) + } +} + +impl Handler for ThresholdKeyshare { + type Result = (); + fn handle( + &mut self, + msg: DecryptionKeySharedCollectionFailed, + ctx: &mut Self::Context, + ) -> Self::Result { + trap(EType::KeyGeneration, &self.bus.clone(), || { + warn!( + e3_id = %msg.e3_id, + missing_parties = ?msg.missing_parties, + "DecryptionKeyShared collection failed: {}", + msg.reason + ); + + self.decryption_key_shared_collector = None; + + self.bus.publish_without_context(E3Failed { + e3_id: msg.e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DecryptionTimeout, + })?; + + ctx.stop(); + Ok(()) + }) + } +} + +impl Handler for ThresholdKeyshare { + type Result = (); + fn handle(&mut self, _: E3RequestComplete, ctx: &mut Self::Context) -> Self::Result { + self.encryption_key_collector = None; + self.decryption_key_collector = None; + self.decryption_key_shared_collector = None; + self.pending = PendingKeyshareWork::default(); + self.notify_sync(ctx, Die); + } +} + +impl Handler for ThresholdKeyshare { + type Result = (); + fn handle(&mut self, _: Die, ctx: &mut Self::Context) -> Self::Result { + warn!("ThresholdKeyshare is shutting down"); + ctx.stop(); + } +} diff --git a/crates/keyshare/src/domain/keyshare_state.rs b/crates/keyshare/src/threshold_keyshare/state.rs similarity index 67% rename from crates/keyshare/src/domain/keyshare_state.rs rename to crates/keyshare/src/threshold_keyshare/state.rs index 8c11197031..4d3ea4aa0e 100644 --- a/crates/keyshare/src/domain/keyshare_state.rs +++ b/crates/keyshare/src/threshold_keyshare/state.rs @@ -335,185 +335,5 @@ impl TryInto for ThresholdKeyshareState { } #[cfg(test)] -mod tests { - use super::*; - use e3_events::E3id; - - fn arc(bytes: &[u8]) -> ArcBytes { - ArcBytes::from_bytes(bytes) - } - - fn base_state(state: KeyshareState) -> ThresholdKeyshareState { - ThresholdKeyshareState::new( - E3id::new("42", 1), - 0, - state, - 1, - 3, - arc(b"params"), - "0xabc".to_string(), - true, - ) - } - - #[test] - fn new_initialises_defaults_and_records_dkg_start() { - let s = base_state(KeyshareState::Init); - assert_eq!(s.variant_name(), "Init"); - assert!(s.aggregated_pk.is_none()); - assert!(s.expelled_parties.is_empty()); - assert!(s.honest_parties.is_none()); - assert!(s.dkg_started_at_unix_secs.is_some()); - assert!(s.proof_aggregation_enabled); - assert_eq!(s.get_threshold_m(), 1); - assert_eq!(s.get_threshold_n(), 3); - assert_eq!(s.get_party_id(), 0); - assert_eq!(s.get_address(), "0xabc"); - } - - #[test] - fn same_branch_transition_is_always_valid() { - // Re-entering the same phase variant must be accepted (idempotent mutations). - let a = KeyshareState::AggregatingDecryptionKey(adk()); - let b = KeyshareState::AggregatingDecryptionKey(adk()); - assert!(a.next(b).is_ok()); - } - - #[test] - fn full_happy_path_transitions_in_order() { - let order = [ - KeyshareState::Init, - KeyshareState::CollectingEncryptionKeys(cek()), - KeyshareState::GeneratingThresholdShare(gts()), - KeyshareState::AggregatingDecryptionKey(adk()), - KeyshareState::ReadyForDecryption(rfd()), - KeyshareState::Decrypting(decrypting()), - KeyshareState::GeneratingDecryptionProof(gdp()), - KeyshareState::Completed, - ]; - for pair in order.windows(2) { - assert!( - pair[0].next(pair[1].clone()).is_ok(), - "expected {} -> {} to be valid", - pair[0].variant_name(), - pair[1].variant_name() - ); - } - } - - #[test] - fn skipping_a_phase_is_rejected() { - let init = KeyshareState::Init; - // Init must go to CollectingEncryptionKeys, not straight to aggregation. - assert!(init - .next(KeyshareState::AggregatingDecryptionKey(adk())) - .is_err()); - } - - #[test] - fn backwards_transition_is_rejected() { - let completed = KeyshareState::Completed; - assert!(completed.next(KeyshareState::Init).is_err()); - } - - #[test] - fn new_state_preserves_metadata_and_advances_phase() { - let s = base_state(KeyshareState::Init); - let next = s - .clone() - .new_state(KeyshareState::CollectingEncryptionKeys(cek())) - .expect("valid transition"); - assert_eq!(next.variant_name(), "CollectingEncryptionKeys"); - assert_eq!(next.e3_id, s.e3_id); - assert_eq!(next.party_id, s.party_id); - assert_eq!(next.threshold_m, s.threshold_m); - assert_eq!(next.threshold_n, s.threshold_n); - } - - #[test] - fn new_state_rejects_illegal_transition() { - let s = base_state(KeyshareState::Init); - assert!(s.new_state(KeyshareState::Completed).is_err()); - } - - // ---- builders for phase data (minimal, transition logic ignores contents) ---- - - fn sens() -> SensitiveBytes { - SensitiveBytes::from_encrypted(&[]) - } - - fn cek() -> CollectingEncryptionKeysData { - CollectingEncryptionKeysData { - sk_bfv: sens(), - pk_bfv: arc(b"pk"), - ciphernode_selected: CiphernodeSelected::default(), - } - } - - fn gts() -> GeneratingThresholdShareData { - GeneratingThresholdShareData { - pk_share: None, - sk_sss: None, - esi_sss: None, - e_sm_raw: None, - sk_bfv: sens(), - pk_bfv: arc(b"pk"), - collected_encryption_keys: Vec::new(), - ciphernode_selected: None, - proof_request_data: None, - } - } - - fn adk() -> AggregatingDecryptionKey { - AggregatingDecryptionKey { - pk_share: arc(b"pk"), - sk_bfv: sens(), - own_sk_share_raw: sens(), - own_esi_shares_raw: Vec::new(), - signed_pk_generation_proof: None, - signed_sk_share_computation_proof: None, - signed_e_sm_share_computation_proof: None, - signed_sk_share_encryption_proofs: Vec::new(), - signed_e_sm_share_encryption_proofs: Vec::new(), - } - } - - fn rfd() -> ReadyForDecryption { - ReadyForDecryption { - pk_share: arc(b"pk"), - sk_poly_sum: sens(), - es_poly_sum: Vec::new(), - signed_pk_generation_proof: None, - signed_sk_share_computation_proof: None, - signed_e_sm_share_computation_proof: None, - signed_sk_share_encryption_proofs: Vec::new(), - signed_e_sm_share_encryption_proofs: Vec::new(), - } - } - - fn decrypting() -> Decrypting { - Decrypting { - pk_share: arc(b"pk"), - sk_poly_sum: sens(), - es_poly_sum: Vec::new(), - ciphertext_output: Vec::new(), - signed_pk_generation_proof: None, - signed_sk_share_computation_proof: None, - signed_e_sm_share_computation_proof: None, - signed_sk_share_encryption_proofs: Vec::new(), - signed_e_sm_share_encryption_proofs: Vec::new(), - } - } - - fn gdp() -> GeneratingDecryptionProof { - GeneratingDecryptionProof { - pk_share: arc(b"pk"), - decryption_share: Vec::new(), - signed_pk_generation_proof: None, - signed_sk_share_computation_proof: None, - signed_e_sm_share_computation_proof: None, - signed_sk_share_encryption_proofs: Vec::new(), - signed_e_sm_share_encryption_proofs: Vec::new(), - } - } -} +#[path = "state_tests.rs"] +mod tests; diff --git a/crates/keyshare/src/threshold_keyshare/state_tests.rs b/crates/keyshare/src/threshold_keyshare/state_tests.rs new file mode 100644 index 0000000000..da18bc72b5 --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/state_tests.rs @@ -0,0 +1,186 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use e3_events::E3id; + +fn arc(bytes: &[u8]) -> ArcBytes { + ArcBytes::from_bytes(bytes) +} + +fn base_state(state: KeyshareState) -> ThresholdKeyshareState { + ThresholdKeyshareState::new( + E3id::new("42", 1), + 0, + state, + 1, + 3, + arc(b"params"), + "0xabc".to_string(), + true, + ) +} + +#[test] +fn new_initialises_defaults_and_records_dkg_start() { + let s = base_state(KeyshareState::Init); + assert_eq!(s.variant_name(), "Init"); + assert!(s.aggregated_pk.is_none()); + assert!(s.expelled_parties.is_empty()); + assert!(s.honest_parties.is_none()); + assert!(s.dkg_started_at_unix_secs.is_some()); + assert!(s.proof_aggregation_enabled); + assert_eq!(s.get_threshold_m(), 1); + assert_eq!(s.get_threshold_n(), 3); + assert_eq!(s.get_party_id(), 0); + assert_eq!(s.get_address(), "0xabc"); +} + +#[test] +fn same_branch_transition_is_always_valid() { + // Re-entering the same phase variant must be accepted (idempotent mutations). + let a = KeyshareState::AggregatingDecryptionKey(adk()); + let b = KeyshareState::AggregatingDecryptionKey(adk()); + assert!(a.next(b).is_ok()); +} + +#[test] +fn full_happy_path_transitions_in_order() { + let order = [ + KeyshareState::Init, + KeyshareState::CollectingEncryptionKeys(cek()), + KeyshareState::GeneratingThresholdShare(gts()), + KeyshareState::AggregatingDecryptionKey(adk()), + KeyshareState::ReadyForDecryption(rfd()), + KeyshareState::Decrypting(decrypting()), + KeyshareState::GeneratingDecryptionProof(gdp()), + KeyshareState::Completed, + ]; + for pair in order.windows(2) { + assert!( + pair[0].next(pair[1].clone()).is_ok(), + "expected {} -> {} to be valid", + pair[0].variant_name(), + pair[1].variant_name() + ); + } +} + +#[test] +fn skipping_a_phase_is_rejected() { + let init = KeyshareState::Init; + // Init must go to CollectingEncryptionKeys, not straight to aggregation. + assert!(init + .next(KeyshareState::AggregatingDecryptionKey(adk())) + .is_err()); +} + +#[test] +fn backwards_transition_is_rejected() { + let completed = KeyshareState::Completed; + assert!(completed.next(KeyshareState::Init).is_err()); +} + +#[test] +fn new_state_preserves_metadata_and_advances_phase() { + let s = base_state(KeyshareState::Init); + let next = s + .clone() + .new_state(KeyshareState::CollectingEncryptionKeys(cek())) + .expect("valid transition"); + assert_eq!(next.variant_name(), "CollectingEncryptionKeys"); + assert_eq!(next.e3_id, s.e3_id); + assert_eq!(next.party_id, s.party_id); + assert_eq!(next.threshold_m, s.threshold_m); + assert_eq!(next.threshold_n, s.threshold_n); +} + +#[test] +fn new_state_rejects_illegal_transition() { + let s = base_state(KeyshareState::Init); + assert!(s.new_state(KeyshareState::Completed).is_err()); +} + +// ---- builders for phase data (minimal, transition logic ignores contents) ---- + +fn sens() -> SensitiveBytes { + SensitiveBytes::from_encrypted(&[]) +} + +fn cek() -> CollectingEncryptionKeysData { + CollectingEncryptionKeysData { + sk_bfv: sens(), + pk_bfv: arc(b"pk"), + ciphernode_selected: CiphernodeSelected::default(), + } +} + +fn gts() -> GeneratingThresholdShareData { + GeneratingThresholdShareData { + pk_share: None, + sk_sss: None, + esi_sss: None, + e_sm_raw: None, + sk_bfv: sens(), + pk_bfv: arc(b"pk"), + collected_encryption_keys: Vec::new(), + ciphernode_selected: None, + proof_request_data: None, + } +} + +fn adk() -> AggregatingDecryptionKey { + AggregatingDecryptionKey { + pk_share: arc(b"pk"), + sk_bfv: sens(), + own_sk_share_raw: sens(), + own_esi_shares_raw: Vec::new(), + signed_pk_generation_proof: None, + signed_sk_share_computation_proof: None, + signed_e_sm_share_computation_proof: None, + signed_sk_share_encryption_proofs: Vec::new(), + signed_e_sm_share_encryption_proofs: Vec::new(), + } +} + +fn rfd() -> ReadyForDecryption { + ReadyForDecryption { + pk_share: arc(b"pk"), + sk_poly_sum: sens(), + es_poly_sum: Vec::new(), + signed_pk_generation_proof: None, + signed_sk_share_computation_proof: None, + signed_e_sm_share_computation_proof: None, + signed_sk_share_encryption_proofs: Vec::new(), + signed_e_sm_share_encryption_proofs: Vec::new(), + } +} + +fn decrypting() -> Decrypting { + Decrypting { + pk_share: arc(b"pk"), + sk_poly_sum: sens(), + es_poly_sum: Vec::new(), + ciphertext_output: Vec::new(), + signed_pk_generation_proof: None, + signed_sk_share_computation_proof: None, + signed_e_sm_share_computation_proof: None, + signed_sk_share_encryption_proofs: Vec::new(), + signed_e_sm_share_encryption_proofs: Vec::new(), + } +} + +fn gdp() -> GeneratingDecryptionProof { + GeneratingDecryptionProof { + pk_share: arc(b"pk"), + decryption_share: Vec::new(), + signed_pk_generation_proof: None, + signed_sk_share_computation_proof: None, + signed_e_sm_share_computation_proof: None, + signed_sk_share_encryption_proofs: Vec::new(), + signed_e_sm_share_encryption_proofs: Vec::new(), + } +} diff --git a/crates/keyshare/src/threshold_keyshare/tests.rs b/crates/keyshare/src/threshold_keyshare/tests.rs new file mode 100644 index 0000000000..8cef1ff686 --- /dev/null +++ b/crates/keyshare/src/threshold_keyshare/tests.rs @@ -0,0 +1,173 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use crate::actors::decryption_key_shared_collector::DecryptionKeySharedCollectionFailed; +use actix::{Actor, Addr, Handler}; +use anyhow::Result; +use e3_crypto::Cipher; +use e3_data::{AutoPersist, DataStore, InMemStore, Persistable, Repository}; +use e3_events::{ + hlc_factory::HlcFactory, BusHandle, E3Stage, E3id, EventBus, EventBusConfig, FailureReason, + HistoryCollector, InterfoldEvent, InterfoldEventData, Sequencer, StoreEventRequested, + StoreEventResponse, TakeEvents, +}; +use e3_fhe_params::DEFAULT_BFV_PRESET; +use std::sync::Arc; + +#[derive(Default)] +struct TestEventStore { + next_seq: u64, +} + +impl Actor for TestEventStore { + type Context = actix::Context; +} + +impl Handler for TestEventStore { + type Result = (); + + fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) -> Self::Result { + let StoreEventRequested { event, sender } = msg; + let seq = self.next_seq; + self.next_seq += 1; + sender.do_send(StoreEventResponse(event.into_sequenced(seq))); + } +} + +fn test_bus() -> (BusHandle, Addr>) { + let event_bus = EventBus::::new(EventBusConfig { deduplicate: true }).start(); + let store = TestEventStore::default().start(); + let sequencer = Sequencer::new(&event_bus, store.recipient()).start(); + let bus = BusHandle::new(event_bus, sequencer, HlcFactory::new()).enable("test-keyshare"); + let history = bus.history(); + (bus, history) +} + +fn test_state() -> Persistable { + let store = InMemStore::new(false).start(); + let repo = Repository::::new(DataStore::from_in_mem(&store)); + repo.send(None) +} + +async fn start_actor() -> Result<( + Addr, + Addr>, + E3id, +)> { + let (bus, history) = test_bus(); + let actor = ThresholdKeyshare::new(ThresholdKeyshareParams { + bus, + cipher: Arc::new(Cipher::from_password("test-password").await?), + state: test_state(), + share_enc_preset: DEFAULT_BFV_PRESET, + }) + .start(); + + Ok((actor, history, E3id::new("42", 1))) +} + +async fn next_event(history: &Addr>) -> Result { + let mut result = history.send(TakeEvents::::new(1)).await?; + assert!(!result.timed_out, "timed out waiting for an event"); + Ok(result.events.pop().expect("expected one event")) +} + +async fn next_events( + history: &Addr>, + count: usize, +) -> Result> { + let result = history + .send(TakeEvents::::new(count)) + .await?; + assert!(!result.timed_out, "timed out waiting for events"); + assert_eq!(result.events.len(), count, "expected {count} events"); + Ok(result.events) +} + +#[actix::test] +async fn encryption_key_collection_failure_preserves_telemetry_and_emits_e3_failed() -> Result<()> { + let (actor, history, e3_id) = start_actor().await?; + let failure = EncryptionKeyCollectionFailed { + e3_id, + reason: "missing encryption keys".to_string(), + missing_parties: vec![2, 3], + }; + + actor.send(failure.clone()).await?; + + let mut events = next_events(&history, 2).await?; + let event = events.remove(0); + assert!(matches!( + event.into_data(), + InterfoldEventData::EncryptionKeyCollectionFailed(data) if data == failure + )); + + let event = events.remove(0); + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == failure.e3_id + && data.failed_at_stage == E3Stage::CommitteeFinalized + && data.reason == FailureReason::DKGTimeout + )); + + Ok(()) +} + +#[actix::test] +async fn threshold_share_collection_failure_preserves_telemetry_and_emits_e3_failed() -> Result<()> +{ + let (actor, history, e3_id) = start_actor().await?; + let failure = ThresholdShareCollectionFailed { + e3_id, + reason: "missing threshold shares".to_string(), + missing_parties: vec![4, 5], + }; + + actor.send(failure.clone()).await?; + + let mut events = next_events(&history, 2).await?; + let event = events.remove(0); + assert!(matches!( + event.into_data(), + InterfoldEventData::ThresholdShareCollectionFailed(data) if data == failure + )); + + let event = events.remove(0); + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == failure.e3_id + && data.failed_at_stage == E3Stage::CommitteeFinalized + && data.reason == FailureReason::DKGTimeout + )); + + Ok(()) +} + +#[actix::test] +async fn decryption_key_shared_collection_failure_emits_e3_failed() -> Result<()> { + let (actor, history, e3_id) = start_actor().await?; + let failure = DecryptionKeySharedCollectionFailed { + e3_id, + reason: "missing decryption key shares".to_string(), + missing_parties: vec![6, 7], + }; + + actor.send(failure.clone()).await?; + + let event = next_event(&history).await?; + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == failure.e3_id + && data.failed_at_stage == E3Stage::CommitteeFinalized + && data.reason == FailureReason::DecryptionTimeout + )); + + Ok(()) +} diff --git a/crates/keyshare/src/actors/threshold_share_collector.rs b/crates/keyshare/src/threshold_keyshare/threshold_share_collector.rs similarity index 100% rename from crates/keyshare/src/actors/threshold_share_collector.rs rename to crates/keyshare/src/threshold_keyshare/threshold_share_collector.rs diff --git a/crates/keyshare/src/domain/timeout_policy.rs b/crates/keyshare/src/threshold_keyshare/timeout_policy.rs similarity index 100% rename from crates/keyshare/src/domain/timeout_policy.rs rename to crates/keyshare/src/threshold_keyshare/timeout_policy.rs diff --git a/crates/net/src/actors.rs b/crates/net/src/actors.rs new file mode 100644 index 0000000000..5a809b8840 --- /dev/null +++ b/crates/net/src/actors.rs @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Compatibility view of thin actors stored with their network capabilities. + +#[path = "document_publishing/actor.rs"] +mod document_publisher; +#[path = "event_conversion/actor.rs"] +mod event_converter; +#[path = "event_buffer/actor.rs"] +mod net_event_buffer; +#[path = "event_translation/actor.rs"] +mod net_event_translator; +#[path = "network_sync/actor.rs"] +mod net_sync_manager; + +pub use document_publisher::{ + handle_document_published_notification, handle_publish_document_requested, DocumentPublisher, +}; +pub use event_converter::EventConverter; +pub use net_event_buffer::{ + NetEventBufferHandle, DEFAULT_MAX_BUFFERED_NET_BYTES, DEFAULT_MAX_BUFFERED_NET_EVENTS, +}; +pub use net_event_translator::NetEventTranslator; + +// Internal wiring helpers used by `setup_net`; not part of the public API. +pub(crate) use net_event_buffer::NetEventBuffer; +pub(crate) use net_sync_manager::NetSyncManager; diff --git a/crates/net/src/actors/document_publisher.rs b/crates/net/src/actors/document_publisher.rs deleted file mode 100644 index 8c529eaf6a..0000000000 --- a/crates/net/src/actors/document_publisher.rs +++ /dev/null @@ -1,1007 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::{ - domain::{ - datetime_to_instant_from_now, DocumentPublishingService, EventConversionService, - IncomingDocument, - }, - events::{ - call_and_await_response, DocumentPublishedNotification, GossipData, NetCommand, NetEvent, - }, - ContentHash, -}; -use actix::prelude::*; -use anyhow::{Context, Result}; -use e3_events::{ - prelude::*, trap, trap_fut, BusHandle, CiphernodeSelected, CorrelationId, DecryptionKeyShared, - DocumentReceived, E3RequestComplete, E3id, EType, EncryptionKeyCreated, EventSource, EventType, - InterfoldEvent, InterfoldEventData, PartyId, PublishDocumentRequested, ThresholdShareCreated, - TypedEvent, -}; -use e3_utils::ArcBytes; -use e3_utils::NotifySync; -use e3_utils::{ - retry::{retry_with_backoff, to_retry}, - MAILBOX_LIMIT, -}; -use futures::TryFutureExt; -use std::{collections::HashMap, sync::Arc, time::Duration}; -use tokio::sync::{broadcast, mpsc}; -use tracing::{debug, info}; - -const KADEMLIA_PUT_TIMEOUT: Duration = Duration::from_secs(30); -const KADEMLIA_GET_TIMEOUT: Duration = Duration::from_secs(30); -const KADEMLIA_BROADCAST_TIMEOUT: Duration = Duration::from_secs(30); - -/// DocumentPublisher is an actor that monitors events from both the Libp2pNetInterface and the -/// Interfold EventBus in order to manage document publishing interactions. The decision/state logic -/// lives in [`DocumentPublishingService`]; this actor only wires events to that service and -/// performs the resulting libp2p/Kademlia I/O. -pub struct DocumentPublisher { - /// Interfold EventBus - bus: BusHandle, - /// NetCommand sender to forward commands to the Libp2pNetInterface - tx: mpsc::Sender, - /// NetEvent receiver to resubscribe for events from the Libp2pNetInterface. This is in an Arc - /// so that we do not do excessive resubscribes without actually listening for events. - rx: Arc>, - /// The gossipsub broadcast topic - topic: String, - /// Pure decision/state service. - service: DocumentPublishingService, -} - -impl DocumentPublisher { - /// Create a new DocumentPublisher actor - pub fn new( - bus: &BusHandle, - tx: &mpsc::Sender, - rx: &Arc>, - topic: impl Into, - ) -> Self { - Self { - bus: bus.clone(), - tx: tx.clone(), - rx: rx.clone(), - topic: topic.into(), - service: DocumentPublishingService::new(), - } - } - - /// This is needed to create simulation libp2p event routers - pub fn is_document_publisher_event(event: &InterfoldEvent) -> bool { - // Add a list of events with paylods for the DHT - matches!( - event.get_data(), - InterfoldEventData::PublishDocumentRequested(_) - | InterfoldEventData::ThresholdShareCreated(_) - | InterfoldEventData::EncryptionKeyCreated(_) - | InterfoldEventData::DecryptionKeyShared(_) - ) - } - - /// Setup the DocumentPublisher and start listening for GossipEvents - pub fn setup( - bus: &BusHandle, - tx: &mpsc::Sender, - rx: &Arc>, - topic: impl Into, - ) -> Addr { - let mut events = rx.resubscribe(); - let addr = Self::new(bus, tx, rx, topic).start(); - EventConverter::setup(bus); - // Listen on all events - bus.subscribe(EventType::All, addr.clone().recipient()); - - // Forward gossip data from NetEvent - tokio::spawn({ - debug!("Spawning event receive loop!"); - let addr = addr.clone(); - async move { - while let Some(event) = - super::recv_net_event(&mut events, "DocumentPublisher").await - { - debug!("Received event {:?}", event); - if let NetEvent::GossipData(GossipData::DocumentPublishedNotification(data)) = - event - { - if let Err(error) = addr.send(data).await { - tracing::warn!( - %error, - "DocumentPublisher stopped; ending DHT notification ingress" - ); - break; - } - } - } - } - }); - - addr - } - - fn handle_ciphernode_selected(&mut self, event: CiphernodeSelected) -> Result<()> { - let CiphernodeSelected { - e3_id, party_id, .. - } = event; - self.service.register_interest(e3_id, party_id); - Ok(()) - } - - fn handle_e3_request_complete(&mut self, event: E3RequestComplete) -> Result<()> { - let keys = self.service.complete_e3(&event.e3_id); - if !keys.is_empty() { - info!( - "Pruning {} DHT records for completed E3 {}", - keys.len(), - event.e3_id - ); - let _ = self.tx.try_send(NetCommand::DhtRemoveRecords { keys }); - } - Ok(()) - } -} - -impl Actor for DocumentPublisher { - type Context = actix::Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT) - } -} - -impl Handler for DocumentPublisher { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - match msg { - InterfoldEventData::PublishDocumentRequested(data) => { - ctx.notify(TypedEvent::new(data, ec)) - } - InterfoldEventData::CiphernodeSelected(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::E3RequestComplete(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - _ => (), - } - } -} - -impl Handler> for DocumentPublisher { - type Result = ResponseFuture<()>; - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - let tx = self.tx.clone(); - let (msg, ec) = msg.into_components(); - - self.service - .track_published_key(&msg.meta.e3_id, &msg.value); - - let rx = self.rx.clone(); - let bus = self.bus.clone(); - let topic = self.topic.clone(); - trap_fut( - EType::IO, - &bus.with_ec(&ec), - handle_publish_document_requested(tx, rx, msg, topic, bus), - ) - } -} - -impl Handler> for DocumentPublisher { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - let (msg, ec) = msg.into_components(); - trap(EType::DocumentPublishing, &self.bus.with_ec(&ec), || { - self.handle_ciphernode_selected(msg) - }) - } -} - -impl Handler> for DocumentPublisher { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - let (msg, ec) = msg.into_components(); - trap(EType::DocumentPublishing, &self.bus.with_ec(&ec), || { - self.handle_e3_request_complete(msg) - }) - } -} - -/// Receiving DocumentPublishedNotification from libp2p -impl Handler for DocumentPublisher { - type Result = ResponseFuture<()>; - fn handle( - &mut self, - msg: DocumentPublishedNotification, - _: &mut Self::Context, - ) -> Self::Result { - let ids = self.service.interest_snapshot(); - let bus = self.bus.clone(); - let tx = self.tx.clone(); - let rx = self.rx.clone(); - trap_fut( - EType::IO, - &bus, - handle_document_published_notification(tx, rx, bus.clone(), ids, msg), - ) - } -} - -/// Called when we receive a PublishDocumentRequested event -pub async fn handle_publish_document_requested( - tx: mpsc::Sender, - rx: Arc>, - event: PublishDocumentRequested, - topic: impl Into, - bus: BusHandle, -) -> Result<()> { - let value = event.value; - let key = ContentHash::from_content(&value); - let expires = Some( - datetime_to_instant_from_now(event.meta.expires_at) - .context("refusing to publish an expired DHT document")?, - ); - - retry_with_backoff( - || { - put_record(tx.clone(), rx.clone(), expires, value.clone(), key.clone()) - .map_err(to_retry) - }, - 4, - 1000, - ) - .await?; - let notification = DocumentPublishedNotification::new(event.meta, key, bus.ts()?); - broadcast_document_published_notification(tx, rx, notification, topic).await?; - Ok(()) -} - -/// Called when we receive a notification from the net_interface -pub async fn handle_document_published_notification( - net_cmds: mpsc::Sender, - net_events: Arc>, - bus: BusHandle, - ids: HashMap, - event: DocumentPublishedNotification, -) -> Result<()> { - let Some(party_id) = DocumentPublishingService::interest_in(&ids, &event) else { - debug!("Node not interested in id {}", event.meta.e3_id); - return Ok(()); - }; - - debug!( - "interested in document {:?} with party_id={:?}", - event, party_id - ); - - let value = retry_with_backoff( - || get_record(net_cmds.clone(), net_events.clone(), event.key.clone()).map_err(to_retry), - 4, - 1000, - ) - .await?; - - // The gossiped metadata is not covered by the DHT content hash. Bind it to the decoded - // payload before persisting DocumentReceived; otherwise a notification for an E3 this node is - // interested in can inject a content-addressed document for a different E3 or party route. - EventConversionService::validate_received(&event.meta, &value)?; - - debug!("Sending received event..."); - bus.publish_from_remote( - DocumentReceived { - meta: event.meta, - value, - }, - event.ts, - None, - EventSource::Net, - )?; - - Ok(()) -} - -/// Call DhtPutRecord Command on the Libp2pNetInterface and handle the results -async fn put_record( - net_cmds: mpsc::Sender, - net_events: Arc>, - expires: Option, - value: ArcBytes, - key: ContentHash, -) -> Result<()> { - let id = CorrelationId::new(); - call_and_await_response( - net_cmds, - net_events, - NetCommand::DhtPutRecord { - correlation_id: id, - expires, - value, - key, - }, - |event| match event { - NetEvent::DhtPutRecordSucceeded { .. } => Some(Ok(())), - NetEvent::DhtPutRecordError { error, .. } => { - Some(Err(anyhow::anyhow!("DHT put record failed: {:?}", error))) - } - _ => None, - }, - KADEMLIA_PUT_TIMEOUT, - ) - .await -} - -/// Call DhtGetRecord Command on the Libp2pNetInterface and handle the results -async fn get_record( - net_cmds: mpsc::Sender, - net_events: Arc>, - key: ContentHash, -) -> Result { - let id = CorrelationId::new(); - call_and_await_response( - net_cmds, - net_events, - NetCommand::DhtGetRecord { - correlation_id: id, - key, - }, - |event| match event { - NetEvent::DhtGetRecordSucceeded { value, .. } => Some(Ok(value.clone())), - NetEvent::DhtGetRecordError { error, .. } => { - Some(Err(anyhow::anyhow!("DHT get record failed: {:?}", error))) - } - _ => None, - }, - KADEMLIA_GET_TIMEOUT, - ) - .await -} - -/// Broadcasts document published notification on Libp2pNetInterface -async fn broadcast_document_published_notification( - net_cmds: mpsc::Sender, - net_events: Arc>, - payload: DocumentPublishedNotification, - topic: impl Into, -) -> Result<()> { - let id = CorrelationId::new(); - call_and_await_response( - net_cmds, - net_events, - NetCommand::GossipPublish { - topic: topic.into(), - correlation_id: id, - data: GossipData::DocumentPublishedNotification(payload), - }, - |event| match event { - NetEvent::GossipPublished { .. } => Some(Ok(())), - NetEvent::GossipPublishError { error, .. } => { - Some(Err(anyhow::anyhow!("GossipPublished failed: {:?}", error))) - } - _ => None, - }, - KADEMLIA_BROADCAST_TIMEOUT, - ) - .await -} - -/// Converts between internal events and network documents. -/// -/// Responsibilities: -/// - Outgoing: Converts ThresholdShareCreated → party-filtered PublishDocumentRequested -/// - Incoming: Converts DocumentReceived → ThresholdShareCreated/EncryptionKeyCreated -/// -/// The conversion logic lives in [`EventConversionService`]; this actor only publishes the -/// resulting events on the bus. -pub struct EventConverter { - bus: BusHandle, -} - -impl EventConverter { - pub fn new(bus: &BusHandle) -> Self { - Self { bus: bus.clone() } - } - - pub fn setup(bus: &BusHandle) -> Addr { - let addr = Self::new(bus).start(); - bus.subscribe(EventType::ThresholdShareCreated, addr.clone().into()); - bus.subscribe(EventType::EncryptionKeyCreated, addr.clone().into()); - bus.subscribe(EventType::DecryptionKeyShared, addr.clone().into()); - bus.subscribe(EventType::DocumentReceived, addr.clone().into()); - addr - } - - fn handle_threshold_share_created(&self, msg: TypedEvent) -> Result<()> { - let (msg, ctx) = msg.into_components(); - if let Some(request) = EventConversionService::threshold_share_to_request(msg)? { - self.bus.publish(request, ctx)?; - } - Ok(()) - } - - fn handle_encryption_key_created(&self, msg: TypedEvent) -> Result<()> { - let (msg, ctx) = msg.into_components(); - if let Some(request) = EventConversionService::encryption_key_to_request(msg)? { - self.bus.publish(request, ctx)?; - } - Ok(()) - } - - fn handle_decryption_key_shared(&self, msg: TypedEvent) -> Result<()> { - let (msg, ctx) = msg.into_components(); - if let Some(request) = EventConversionService::decryption_key_to_request(msg)? { - self.bus.publish(request, ctx)?; - } - Ok(()) - } - - /// Convert received document to internal events. - /// Note: Filtering already happened in DocumentPublisher before DHT fetch. - fn handle_document_received(&self, msg: TypedEvent) -> Result<()> { - let (msg, ctx) = msg.into_components(); - match EventConversionService::decode_received(&msg.meta, &msg.value)? { - IncomingDocument::ThresholdShare(evt) => { - self.bus.publish(evt, ctx)?; - } - IncomingDocument::EncryptionKey(evt) => { - self.bus.publish(evt, ctx)?; - } - IncomingDocument::DecryptionKey(evt) => { - self.bus.publish(evt, ctx)?; - } - } - Ok(()) - } -} - -impl Actor for EventConverter { - type Context = actix::Context; -} - -impl Handler for EventConverter { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (data, ec) = msg.into_components(); - match data { - InterfoldEventData::ThresholdShareCreated(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::EncryptionKeyCreated(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::DecryptionKeyShared(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::DocumentReceived(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - _ => (), - } - } -} - -impl Handler> for EventConverter { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::DocumentPublishing, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_threshold_share_created(msg), - ) - } -} - -impl Handler> for EventConverter { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::DocumentPublishing, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_encryption_key_created(msg), - ) - } -} - -impl Handler> for EventConverter { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::DocumentPublishing, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_decryption_key_shared(msg), - ) - } -} - -impl Handler> for EventConverter { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap( - EType::DocumentPublishing, - &self.bus.with_ec(msg.get_ctx()), - || self.handle_document_received(msg), - ) - } -} - -#[cfg(test)] -mod tests { - use std::{collections::HashMap, num::NonZero, time::Duration}; - - use super::*; - use crate::events::NetCommand; - use actix::Addr; - use anyhow::{bail, Result}; - use chrono::Utc; - use e3_ciphernode_builder::EventSystem; - use e3_events::{ - BusHandle, CiphernodeSelected, DocumentKind, DocumentMeta, E3id, EncryptionKey, GetEvents, - HistoryCollector, InterfoldError, InterfoldEvent, PublishDocumentRequested, TakeEvents, - }; - use libp2p::kad::{GetRecordError, PutRecordError, RecordKey}; - use std::time::Instant; - use tokio::{ - sync::{broadcast, mpsc}, - time::{sleep, timeout}, - }; - use tracing::subscriber::DefaultGuard; - - #[allow(clippy::type_complexity)] - fn setup_test() -> Result<( - DefaultGuard, - BusHandle, - mpsc::Sender, - mpsc::Receiver, - broadcast::Sender, - Arc>, - Addr>, - Addr>, - Addr, - )> { - use tracing_subscriber::{fmt, EnvFilter}; - - let subscriber = fmt() - .with_env_filter(EnvFilter::new("debug")) - .with_test_writer() - .finish(); - - let guard = tracing::subscriber::set_default(subscriber); - - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("test"); - let (net_cmd_tx, net_cmd_rx) = mpsc::channel(100); - let (net_evt_tx, net_evt_rx) = broadcast::channel(100); - let net_evt_rx = Arc::new(net_evt_rx); - let history = HistoryCollector::::new().start(); - let error = HistoryCollector::::new().start(); - bus.subscribe(EventType::All, history.clone().recipient()); - bus.subscribe(EventType::InterfoldError, error.clone().recipient()); - let publisher = DocumentPublisher::setup(&bus, &net_cmd_tx, &net_evt_rx, "topic"); - - Ok(( - guard, bus, net_cmd_tx, net_cmd_rx, net_evt_tx, net_evt_rx, history, error, publisher, - )) - } - - #[actix::test] - async fn test_publishes_document() -> Result<()> { - let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _, _, _) = - setup_test()?; - let value = ArcBytes::from_bytes(b"I am a special document"); - let expires_at = Some(Utc::now() + chrono::Duration::days(1)); - let e3_id = E3id::new("1243", 1); - - // 1. Send a request to publish - bus.publish_without_context(PublishDocumentRequested { - meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), - value: value.clone(), - })?; - - // 2. Document publisher should have asked the Libp2pNetInterface to put the doc on Kademlia - let Some(NetCommand::DhtPutRecord { - correlation_id, - expires, - value: msg_value, - key, - }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()) - .await - .expect("did not receive DhtPutRecord") - else { - bail!("msg not as expected"); - }; - - // Fake DHT put the record - let mut mykad: HashMap> = HashMap::new(); - mykad.insert(key.clone(), msg_value.extract_bytes()); - - // 3. Report that everything went well - net_evt_tx.send(NetEvent::DhtPutRecordSucceeded { - correlation_id, - key, - })?; - - // 4. Expect a DocumentPublishedNotification to have been emitted - let Some(NetCommand::GossipPublish { - topic, - correlation_id, - data: GossipData::DocumentPublishedNotification(notification), - .. - }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()) - .await - .expect("did not receive GossipPublish") - else { - bail!("msg not as expected"); - }; - - // 5. Report everything went well - net_evt_tx.send(NetEvent::GossipPublished { - correlation_id, - message_id: libp2p::gossipsub::MessageId::new(&[1, 2, 3]), - })?; - - assert_eq!(topic, "topic"); - assert_eq!(notification.meta.e3_id, E3id::new("1243", 1)); - - assert_eq!( - mykad.get(¬ification.key), - Some(&b"I am a special document".to_vec()), - "value was not correct" - ); - - assert!( - is_between(expires.unwrap(), days_from_now(0), days_from_now(1)), - "Expiry was not set" - ); - - Ok(()) - } - - #[actix::test] - async fn expired_document_is_rejected_without_a_dht_write() -> Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("expired-document"); - let (net_cmd_tx, mut net_cmd_rx) = mpsc::channel(1); - let (_net_evt_tx, net_evt_rx) = broadcast::channel(1); - let event = PublishDocumentRequested { - meta: DocumentMeta::new( - E3id::new("expired", 1), - DocumentKind::TrBFV, - vec![], - Some(Utc::now() - chrono::Duration::seconds(1)), - ), - value: ArcBytes::from_bytes(b"stale"), - }; - - let error = handle_publish_document_requested( - net_cmd_tx, - Arc::new(net_evt_rx), - event, - "topic", - bus, - ) - .await - .unwrap_err(); - - assert!(format!("{error:#}").contains("expiry is not in the future")); - assert!(matches!( - net_cmd_rx.try_recv(), - Err(mpsc::error::TryRecvError::Empty | mpsc::error::TryRecvError::Disconnected) - )); - Ok(()) - } - - #[actix::test] - async fn test_get_document_fails_with_exponential_backoff() -> Result<()> { - let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _, errors, _) = - setup_test()?; - - let value = b"I am a special document".to_vec(); - let expires_at = Some(Utc::now() + chrono::Duration::days(1)); - let e3_id = E3id::new("1243", 1); - let cid = ContentHash::from_content(&value); - - // 1. Ensure the publisher is interested in the id by receiving CiphernodeSelected - bus.publish_without_context(CiphernodeSelected { - e3_id: e3_id.clone(), - threshold_m: 3, - threshold_n: 5, - ..CiphernodeSelected::default() - })?; - - net_evt_tx.send(NetEvent::GossipData( - GossipData::DocumentPublishedNotification(DocumentPublishedNotification { - key: cid.clone(), - meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), - ts: 123, - }), - ))?; - - for _ in 0..4 { - // Expect retry - let Some(NetCommand::DhtGetRecord { correlation_id, .. }) = - timeout(Duration::from_secs(15), net_cmd_rx.recv()) - .await - .expect("did not receive DhtGetRecord") - else { - bail!("msg not as expected"); - }; - - // Report failure - net_evt_tx.send(NetEvent::DhtGetRecordError { - correlation_id, - error: GetRecordError::Timeout { - key: RecordKey::new(&cid), - }, - })?; - } - - // wait for events to settle - let errors = errors.send(TakeEvents::new(1)).await?; - let error: InterfoldError = errors.events.first().unwrap().try_into()?; - assert_eq!( - error.message, - "Operation failed after 4 attempts. Last error: DHT get record failed: Timeout { key: Key(b\"\\xda-\\xe1\\xc0T\\x11$X\\x05\\xd1\\xd4\\xa6C\\x86\\x96\\xb7e\\xd9j\\x96\\x1bD\\xc8P#\\x0f\\\"\\xea A@b\") }" - ); - - Ok(()) - } - - #[actix::test] - async fn test_publishes_document_fails_with_exponential_backoff() -> Result<()> { - let ( - _guard, - bus, - _net_cmd_tx, - mut net_cmd_rx, - net_evt_tx, - _net_evt_rx, - _history, - errors, - _, - ) = setup_test()?; - let value = ArcBytes::from_bytes(b"I am a special document"); - let expires_at = Some(Utc::now() + chrono::Duration::days(1)); - let e3_id = E3id::new("1243", 1); - - // Send a request to publish - bus.publish_without_context(PublishDocumentRequested { - meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), - value: value.clone(), - })?; - - for _ in 0..4 { - // Expect retry - let Some(NetCommand::DhtPutRecord { correlation_id, .. }) = - timeout(Duration::from_secs(15), net_cmd_rx.recv()) - .await - .expect("did not receive DhtPutRecord") - else { - bail!("msg not as expected"); - }; - - // Report failure - net_evt_tx.send(NetEvent::DhtPutRecordError { - correlation_id, - error: crate::events::PutOrStoreError::PutRecordError( - PutRecordError::QuorumFailed { - key: RecordKey::new(b"I got the secret"), - success: vec![], - quorum: NonZero::new(1).unwrap(), - }, - ), - })?; - } - - // Expect error to exist - let errors = errors.send(TakeEvents::new(1)).await?; - let error: InterfoldError = errors.events.first().unwrap().try_into()?; - assert_eq!( - error.message, - "Operation failed after 4 attempts. Last error: DHT put record failed: PutRecordError(QuorumFailed { key: Key(b\"I got the secret\"), success: [], quorum: 1 })" - ); - - Ok(()) - } - - #[actix::test] - async fn test_notified_of_document() -> Result<()> { - let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, history, _, _) = - setup_test()?; - - let expires_at = Utc::now() + chrono::Duration::days(1); - let e3_id = E3id::new("1243", 1); - let value = EventConversionService::encryption_key_to_request(EncryptionKeyCreated { - e3_id: e3_id.clone(), - key: Arc::new(EncryptionKey::new(1, ArcBytes::from_bytes(b"public key"))), - external: false, - })? - .expect("local key should produce a document") - .value; - let cid = ContentHash::from_content(&value); - - // 1. Ensure the publisher is interested in the id by receiving CiphernodeSelected - bus.publish_without_context(CiphernodeSelected { - e3_id: e3_id.clone(), - threshold_m: 3, - threshold_n: 5, - ..CiphernodeSelected::default() - })?; - - // 2. Dispatch a NetEvent from the Libp2pNetInterface signaling that a document was published - net_evt_tx.send(NetEvent::GossipData( - GossipData::DocumentPublishedNotification(DocumentPublishedNotification { - key: ContentHash::from_content(b"wrong document".as_ref()), - meta: DocumentMeta::new( - E3id::new("1111", 1), - DocumentKind::TrBFV, - vec![], - Some(expires_at), - ), - ts: 123, - }), - ))?; - - // 3. Nothing happens... - let result = timeout(Duration::from_secs(1), net_cmd_rx.recv()).await; - assert!(result.is_err(), "Expected timeout but received a message"); - - // 4. Dispatch a NetEvent from the Libp2pNetInterface signaling that a document we ARE interested - // in was published - net_evt_tx.send(NetEvent::GossipData( - GossipData::DocumentPublishedNotification(DocumentPublishedNotification { - key: cid.clone(), - meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], Some(expires_at)), - ts: 100, - }), - ))?; - - // 5. Expect that DocumentPublisher will make a DhtGetRecord request - let Some(NetCommand::DhtGetRecord { - key, - correlation_id, - }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()) - .await - .expect("did not receive DhtGetRecord") - else { - bail!("msg not as expected"); - }; - - assert_eq!(key, cid); - - // 6. Forward the document - net_evt_tx.send(NetEvent::DhtGetRecordSucceeded { - key: cid, - correlation_id, - value: value.clone(), - })?; - - // wait for events to settle - sleep(Duration::from_millis(100)).await; - - // Check event was dispatched - let events = history.send(GetEvents::new()).await?; - let Some(InterfoldEventData::DocumentReceived(DocumentReceived { value: doc, .. })) = - events.iter().find_map(|event| match event.get_data() { - data @ InterfoldEventData::DocumentReceived(_) => Some(data), - _ => None, - }) - else { - bail!("No event sent"); - }; - - assert_eq!( - doc.extract_bytes(), - value.extract_bytes(), - "document did not match" - ); - - Ok(()) - } - - #[actix::test] - async fn notification_cannot_relabel_payload_for_another_e3() -> Result<()> { - let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _rx, history, errors, _) = - setup_test()?; - let interested_e3 = E3id::new("100", 1); - let payload_e3 = E3id::new("200", 1); - let value = EventConversionService::encryption_key_to_request(EncryptionKeyCreated { - e3_id: payload_e3, - key: Arc::new(EncryptionKey::new(1, ArcBytes::from_bytes(b"public key"))), - external: false, - })? - .expect("local key should produce a document") - .value; - let key = ContentHash::from_content(&value); - - bus.publish_without_context(CiphernodeSelected { - e3_id: interested_e3.clone(), - threshold_m: 3, - threshold_n: 5, - ..CiphernodeSelected::default() - })?; - net_evt_tx.send(NetEvent::GossipData( - GossipData::DocumentPublishedNotification(DocumentPublishedNotification { - key: key.clone(), - meta: DocumentMeta::new( - interested_e3, - DocumentKind::TrBFV, - vec![], - Some(Utc::now() + chrono::Duration::days(1)), - ), - ts: 100, - }), - ))?; - - let Some(NetCommand::DhtGetRecord { correlation_id, .. }) = - timeout(Duration::from_secs(1), net_cmd_rx.recv()) - .await - .expect("did not receive DhtGetRecord") - else { - bail!("msg not as expected"); - }; - net_evt_tx.send(NetEvent::DhtGetRecordSucceeded { - key, - correlation_id, - value, - })?; - - let error_events = errors.send(TakeEvents::new(1)).await?; - let error: InterfoldError = error_events.events.first().unwrap().try_into()?; - assert!(error.message.contains("metadata E3 1:100")); - assert!(error.message.contains("payload E3 1:200")); - - let events = history.send(GetEvents::new()).await?; - assert!( - !events - .iter() - .any(|event| matches!(event.get_data(), InterfoldEventData::DocumentReceived(_))), - "mismatched document must not be persisted" - ); - Ok(()) - } - - pub fn is_between(instant: Instant, start: Instant, end: Instant) -> bool { - let (min, max) = if start <= end { - (start, end) - } else { - (end, start) - }; - instant >= min && instant <= max - } - - pub fn days_from_now(days: u64) -> Instant { - Instant::now() + Duration::from_secs(60 * 60 * 24 * days) - } -} diff --git a/crates/net/src/actors/net_event_buffer.rs b/crates/net/src/actors/net_event_buffer.rs deleted file mode 100644 index c155f2871a..0000000000 --- a/crates/net/src/actors/net_event_buffer.rs +++ /dev/null @@ -1,330 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use actix::{Actor, ActorContext, AsyncContext, Handler, Message}; -use anyhow::{anyhow, Context, Result}; -use e3_events::{ - BusHandle, EType, ErrorDispatcher, Event, EventSubscriber, EventType, InterfoldEvent, - InterfoldEventData, -}; -use e3_utils::MAILBOX_LIMIT; -use tokio::sync::broadcast::{self, error::RecvError}; -use tokio::sync::oneshot; - -use crate::domain::net_buffer::{BufferDecision, NetEventBufferState}; -use crate::events::NetEvent; - -pub const DEFAULT_MAX_BUFFERED_NET_EVENTS: usize = 1_024; -pub const DEFAULT_MAX_BUFFERED_NET_BYTES: usize = 256 * 1024 * 1024; - -pub struct NetEventBufferHandle { - readiness: oneshot::Receiver>, -} - -impl NetEventBufferHandle { - pub async fn wait_until_running(self) -> Result<()> { - self.readiness - .await - .context("network event buffer stopped before reporting startup status")? - .map_err(anyhow::Error::msg) - } -} - -/// Actor that controls a broadcast channel which will buffer NetEvents until it receives a -/// `SyncEnded` event, at which time it releases all buffered events to the output channel. The -/// buffering decision logic lives in [`NetEventBufferState`]. -pub struct NetEventBuffer { - state: NetEventBufferState, - input_rx: Option>, - output_tx: broadcast::Sender, - bus: BusHandle, - max_events: usize, - max_bytes: usize, - readiness: Option>>, -} - -impl NetEventBuffer { - pub(crate) fn setup_with_limits( - bus: &BusHandle, - input_rx: &broadcast::Receiver, - max_events: usize, - max_bytes: usize, - ) -> (broadcast::Receiver, NetEventBufferHandle) { - let input_rx = input_rx.resubscribe(); - let (output_tx, output_rx) = broadcast::channel(max_events); - let (readiness_tx, readiness) = oneshot::channel(); - - let actor = Self { - state: NetEventBufferState::syncing(), - input_rx: Some(input_rx), - output_tx, - bus: bus.clone(), - max_events, - max_bytes, - readiness: Some(readiness_tx), - }; - - let addr = actor.start(); - - // Subscribe to InterfoldEvent on the bus - bus.subscribe(EventType::SyncEnded, addr.clone().recipient()); - - (output_rx, NetEventBufferHandle { readiness }) - } - - fn handle_interfold_event(&mut self, msg: InterfoldEvent) -> Result<()> { - if let InterfoldEventData::SyncEnded(_) = msg.get_data() { - return self.process_sync_ended(); - } - Ok(()) - } - - fn process_sync_ended(&mut self) -> Result<()> { - let pending = self.state.run()?; - for event in pending { - self.forward_event(event)?; - } - self.signal_startup(Ok(())); - Ok(()) - } - - fn forward_event(&mut self, event: NetEvent) -> Result<()> { - self.output_tx - .send(event) - .map_err(|e| anyhow!("Failed to forward event: {}", e))?; - Ok(()) - } - - fn signal_startup(&mut self, result: std::result::Result<(), String>) { - if let Some(sender) = self.readiness.take() { - let _ = sender.send(result); - } - } - - fn fail_closed(&mut self, error: anyhow::Error, ctx: &mut actix::Context) { - let reason = format!( - "network event buffer failed closed: {error:#}; startup will stop rather than drop \ - live protocol input. Increase the configured buffer only after measuring the sync \ - backlog, or restore peer/RPC health and restart" - ); - self.signal_startup(Err(reason.clone())); - self.bus.err(EType::Net, anyhow!(reason)); - ctx.stop(); - } -} - -impl Actor for NetEventBuffer { - type Context = actix::Context; - - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT); - // Spawn task to read from broadcast channel - let addr = ctx.address(); - let mut input_rx = self.input_rx.take().expect("input_rx should be present"); - - actix::spawn(async move { - loop { - match input_rx.recv().await { - Ok(event) => { - if addr.send(IncomingNetEvent(event)).await.is_err() { - break; - } - } - Err(RecvError::Lagged(skipped)) => { - let _ = addr.send(NetInputLagged(skipped)).await; - break; - } - Err(RecvError::Closed) => break, - } - } - }); - } - - fn stopped(&mut self, _: &mut Self::Context) { - self.signal_startup(Err( - "network event buffer stopped before startup synchronization completed".to_owned(), - )); - } -} - -#[derive(Message)] -#[rtype(result = "()")] -struct IncomingNetEvent(NetEvent); - -#[derive(Message)] -#[rtype(result = "()")] -struct NetInputLagged(u64); - -impl Handler for NetEventBuffer { - type Result = (); - - fn handle(&mut self, msg: IncomingNetEvent, ctx: &mut Self::Context) { - let event_bytes = if self.state.is_running() { - 0 - } else { - msg.0.buffered_size_bytes() - }; - let result = self - .state - .observe(msg.0, event_bytes, self.max_events, self.max_bytes) - .and_then(|decision| match decision { - BufferDecision::Buffered => Ok(()), - BufferDecision::Forward(event) => self.forward_event(event), - }); - if let Err(error) = result { - self.fail_closed(error, ctx); - } - } -} - -impl Handler for NetEventBuffer { - type Result = (); - - fn handle(&mut self, msg: NetInputLagged, ctx: &mut Self::Context) { - self.fail_closed( - anyhow!( - "network event input skipped {} events because its bounded broadcast receiver lagged", - msg.0 - ), - ctx, - ); - } -} - -impl Handler for NetEventBuffer { - type Result = (); - - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - if let Err(error) = self.handle_interfold_event(msg) { - self.fail_closed(error, ctx); - } - } -} - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use super::*; - use crate::events::{GossipData, NetEvent}; - use e3_ciphernode_builder::EventSystem; - use e3_events::EventPublisher; - use e3_events::SyncEnded; - use tokio::{ - sync::broadcast, - time::{sleep, timeout}, - }; - - #[actix::test] - async fn test_buffers_until_sync_ended() -> Result<()> { - // Setup - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("test"); - let (input_tx, input_rx) = broadcast::channel(16); - let (mut output_rx, handle) = NetEventBuffer::setup_with_limits( - &bus, - &input_rx, - DEFAULT_MAX_BUFFERED_NET_EVENTS, - DEFAULT_MAX_BUFFERED_NET_BYTES, - ); - - // Send events while syncing - should be buffered - let event1 = NetEvent::GossipData(GossipData::GossipBytes(vec![1, 2, 3])); - let event2 = NetEvent::GossipData(GossipData::GossipBytes(vec![4, 5, 6])); - input_tx.send(event1.clone()).unwrap(); - input_tx.send(event2.clone()).unwrap(); - - // Give actor time to process - sleep(Duration::from_millis(10)).await; - - // Verify no events forwarded yet (should timeout) - assert!( - timeout(Duration::from_millis(50), output_rx.recv()) - .await - .is_err(), - "Events should be buffered, not forwarded during sync" - ); - - // Send SyncEnded event - bus.publish_without_context(SyncEnded::new()).unwrap(); - handle.wait_until_running().await?; - - // Now buffered events should be forwarded - let received1 = output_rx.recv().await.unwrap(); - let received2 = output_rx.recv().await.unwrap(); - - assert!( - matches!(received1, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![1, 2, 3]) - ); - assert!( - matches!(received2, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![4, 5, 6]) - ); - - // Send new event after sync - should forward immediately - let event3 = NetEvent::GossipData(GossipData::GossipBytes(vec![7, 8, 9])); - input_tx.send(event3.clone()).unwrap(); - - let received3 = - tokio::time::timeout(tokio::time::Duration::from_millis(100), output_rx.recv()) - .await - .expect("Event should be forwarded immediately after sync") - .unwrap(); - - assert!( - matches!(received3, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![7, 8, 9]) - ); - - Ok(()) - } - - #[actix::test] - async fn startup_buffer_overflow_fails_readiness_without_dropping_oldest() -> Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("test-overflow"); - let (input_tx, input_rx) = broadcast::channel(16); - let (_output_rx, handle) = - NetEventBuffer::setup_with_limits(&bus, &input_rx, 1, DEFAULT_MAX_BUFFERED_NET_BYTES); - - input_tx.send(NetEvent::GossipData(GossipData::GossipBytes(vec![1])))?; - input_tx.send(NetEvent::GossipData(GossipData::GossipBytes(vec![2])))?; - - let error = timeout(Duration::from_secs(1), handle.wait_until_running()) - .await - .context("network buffer did not report overflow")? - .expect_err("overflow must fail startup readiness") - .to_string(); - assert!(error.contains("events=1/1"), "{error}"); - assert!( - error.contains("startup will stop rather than drop"), - "{error}" - ); - Ok(()) - } - - #[actix::test] - async fn startup_buffer_enforces_estimated_payload_bytes() -> Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("test-byte-overflow"); - let (input_tx, input_rx) = broadcast::channel(16); - let event = NetEvent::GossipData(GossipData::GossipBytes(vec![0; 32])); - let estimated_bytes = event.buffered_size_bytes(); - let (_output_rx, handle) = - NetEventBuffer::setup_with_limits(&bus, &input_rx, 16, estimated_bytes - 1); - - input_tx.send(event)?; - - let error = timeout(Duration::from_secs(1), handle.wait_until_running()) - .await - .context("network buffer did not report byte overflow")? - .expect_err("byte overflow must fail startup readiness") - .to_string(); - assert!( - error.contains(&format!("next_event_bytes={estimated_bytes}")), - "{error}" - ); - Ok(()) - } -} diff --git a/crates/net/src/actors/net_sync_manager.rs b/crates/net/src/actors/net_sync_manager.rs deleted file mode 100644 index 931bbb43d3..0000000000 --- a/crates/net/src/actors/net_sync_manager.rs +++ /dev/null @@ -1,1068 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use actix::{Actor, Addr, AsyncContext, Handler, Message, Recipient, ResponseFuture}; -use anyhow::{bail, Context, Result}; -use e3_events::{ - prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EventSource, - EventStoreFilter, EventStoreQueryBy, EventStoreQueryResponse, EventType, - HistoricalNetSyncEventsReceived, HistoricalNetSyncStart, InterfoldEvent, InterfoldEventData, - NetReady, TsAgg, TypedEvent, Unsequenced, -}; -use e3_utils::MAILBOX_LIMIT; -use libp2p::PeerId; -use serde::{Deserialize, Serialize}; -use std::{ - collections::{HashMap, HashSet}, - convert::TryInto, - sync::Arc, - time::Duration, -}; -use tokio::sync::{broadcast, mpsc}; -use tracing::{debug, error, info, warn}; - -use crate::{ - direct_requester::DirectRequester, - direct_responder::DirectResponder, - domain::{ - build_sync_batch, - net_event_batch::{ - fetch_all_batched_events_with_budget, FetchEventsSince, SyncFetchBudget, - }, - sync_coordinator::sync_scan_limit, - wire::{decode, MAX_DIRECT_MESSAGE_BYTES}, - EventTranslationService, NetReadiness, ReadinessDecision, SyncBatchOutcome, - }, - events::{ - await_event, GossipData, IncomingRequest, NetCommand, NetEvent, PeerTarget, - ProtocolResponse, - }, -}; - -/// Maximum time to wait for a `ConnectionEstablished` event after all dials -/// failed before publishing `NetReady` anyway. -const NET_READY_CONNECT_TIMEOUT: Duration = Duration::from_secs(60); - -/// Direct-request retry settings for a single historical sync fetch attempt. -const SYNC_FETCH_MAX_RETRIES: u32 = 3; -const SYNC_FETCH_RETRY_TIMEOUT: Duration = Duration::from_secs(5); - -/// If a historical sync fetch fails, wait this long for a fresh connection -/// before retrying anyway against currently connected peers. -const SYNC_RECOVERY_RETRY_INTERVAL: Duration = Duration::from_secs(15); - -/// Number of recovery rounds to try for failed aggregates after the initial fetch pass. -const SYNC_RECOVERY_MAX_ATTEMPTS: usize = 3; - -/// Bound remote work independently of the actor mailbox. Per-peer admission prevents one -/// authenticated transport identity from occupying the global allowance. -const MAX_IN_FLIGHT_SYNC_REQUESTS: usize = 16; -const MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER: usize = 2; - -/// Expire storage requests before libp2p's 30-second request timeout so a failed local query cannot -/// retain a responder and permanently consume one of the bounded in-flight slots. -const INCOMING_SYNC_REQUEST_TIMEOUT: Duration = Duration::from_secs(25); - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SyncResponseValue { - pub events: Vec>, - pub ts: u128, -} - -impl TryInto> for SyncResponseValue { - type Error = anyhow::Error; - - fn try_into(self) -> Result, Self::Error> { - bincode::serialize(&self).context("failed to serialize sync response") - } -} - -impl TryFrom> for SyncResponseValue { - type Error = anyhow::Error; - - fn try_from(value: Vec) -> Result { - decode(&value, MAX_DIRECT_MESSAGE_BYTES).context("failed to deserialize sync response") - } -} - -#[derive(Debug, Clone)] -pub struct SyncRequestSucceeded { - pub response: SyncResponseValue, -} - -struct PendingSyncRequest { - peer: PeerId, - responder: DirectResponder, -} - -pub struct NetSyncManager { - /// Interfold EventBus - bus: BusHandle, - /// NetCommand sender to forward commands to the Libp2pNetInterface - tx: mpsc::Sender, - /// NetEvents receiver to receive events - rx: Arc>, - eventstore: Recipient>, - requests: HashMap, - /// Pure readiness state machine. - readiness: NetReadiness, - /// Gossipsub topic used to re-broadcast our own forwardable artifacts after a restart. - topic: String, - /// Snapshot-cursor map captured from `HistoricalNetSyncStart`. Bounds the post-restart - /// re-broadcast query to the in-flight (un-snapshotted) window. - rebroadcast_since: Option>, - /// Correlation ids of in-flight re-broadcast EventStore queries, so their responses can be - /// distinguished from ordinary sync-request responses. - rebroadcast_query_ids: HashSet, - /// Set once `NetReady` has been published (peers connected or fallback timeout elapsed). - net_ready: bool, - /// Guard so the post-restart re-broadcast fires at most once per process. - rebroadcast_started: bool, -} - -impl NetSyncManager { - pub fn new( - bus: &BusHandle, - tx: &mpsc::Sender, - rx: &Arc>, - eventstore: Recipient>, - topic: &str, - ) -> Self { - Self { - bus: bus.clone(), - tx: tx.clone(), - rx: Arc::clone(rx), - eventstore, - requests: HashMap::new(), - readiness: NetReadiness::new(), - topic: topic.to_string(), - rebroadcast_since: None, - rebroadcast_query_ids: HashSet::new(), - net_ready: false, - rebroadcast_started: false, - } - } - - fn publish_net_ready(&self) -> Result<()> { - info!("NetSyncManager: publishing NetReady"); - self.bus.publish_without_context(NetReady::new())?; - Ok(()) - } - - fn request_capacity_error(&self, peer: &PeerId) -> Option<&'static str> { - if self.requests.len() >= MAX_IN_FLIGHT_SYNC_REQUESTS { - return Some("too many in-flight sync requests"); - } - let peer_requests = self - .requests - .values() - .filter(|request| &request.peer == peer) - .count(); - if peer_requests >= MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER { - return Some("too many in-flight sync requests from this peer"); - } - None - } - - fn expire_sync_request(&mut self, id: CorrelationId) { - let Some(pending) = self.requests.remove(&id) else { - return; - }; - warn!( - peer = %pending.peer, - correlation_id = %id, - timeout_ms = INCOMING_SYNC_REQUEST_TIMEOUT.as_millis(), - "Incoming historical-sync storage query timed out" - ); - if let Err(error) = pending.responder.respond(ProtocolResponse::Error( - "historical sync request timed out".to_string(), - )) { - warn!( - peer = %pending.peer, - correlation_id = %id, - %error, - "Failed to send historical-sync timeout response" - ); - } - } - - /// After a restart, proactively re-gossip this node's own already-produced forwardable DKG - /// artifacts (H3/H11). Resume from a persisted phase is otherwise passive: the restored - /// keyshare/aggregator actors wait for peer documents and never re-emit their own outputs, so - /// peers that missed the original gossip (cache expiry, DHT miss, peer churn) can stall the - /// node to its phase timeout. - /// - /// The artifacts are sent straight to libp2p as `GossipPublish`, bypassing both the EventBus - /// dedup window (which already tracked them during replay) and the translator (which is only - /// created on `EffectsEnabled`). Re-broadcasting the byte-identical original payload is - /// equivocation-safe (peers dedup by event id) and idempotent. The query is bounded to the - /// snapshot-cursor window so only the in-flight (un-delivered) artifacts are re-sent. - fn maybe_rebroadcast_own_artifacts(&mut self, ctx: &mut actix::Context) { - if self.rebroadcast_started || !self.net_ready { - return; - } - let Some(since) = self.rebroadcast_since.clone() else { - return; - }; - self.rebroadcast_started = true; - - let id = CorrelationId::new(); - self.rebroadcast_query_ids.insert(id); - info!("NetSyncManager: querying own forwardable artifacts for post-restart re-broadcast"); - if let Err(e) = self.eventstore.try_send( - EventStoreQueryBy::::new(id, since, ctx.address().recipient()) - .with_filter(EventStoreFilter::Source(EventSource::Local)), - ) { - error!("Failed to query EventStore for re-broadcast: {e}"); - self.rebroadcast_query_ids.remove(&id); - self.rebroadcast_started = false; - } - } - - /// Re-gossip the node's own forwardable artifacts returned by the re-broadcast query. - fn handle_rebroadcast_response(&mut self, events: Vec) { - let mut count = 0usize; - for event in events { - if !EventTranslationService::is_forwardable_event(&event) { - continue; - } - let data: GossipData = match event.try_into() { - Ok(data) => data, - Err(e) => { - warn!("Failed to convert own artifact to gossip data: {e}"); - continue; - } - }; - if let Err(e) = self.tx.try_send(NetCommand::GossipPublish { - topic: self.topic.clone(), - data, - correlation_id: CorrelationId::new(), - }) { - warn!("Failed to re-broadcast own artifact (channel full or closed): {e}"); - } else { - count += 1; - } - } - info!("NetSyncManager: re-broadcast {count} own forwardable artifact(s) after restart"); - } - - /// Apply a readiness decision: publish `NetReady`, or schedule the fallback timeout. - fn apply_readiness(&mut self, decision: ReadinessDecision, ctx: &mut actix::Context) { - match decision { - ReadinessDecision::PublishReady => { - if let Err(e) = self.publish_net_ready() { - error!("Failed to publish NetReady: {e}"); - } - self.net_ready = true; - self.maybe_rebroadcast_own_artifacts(ctx); - } - ReadinessDecision::WaitForConnection => { - info!( - "All peer dials failed, waiting for connections before publishing NetReady..." - ); - ctx.run_later(NET_READY_CONNECT_TIMEOUT, move |this, ctx| { - if let ReadinessDecision::PublishReady = this.readiness.on_connect_timeout() { - warn!("No peer connections established within 60s timeout, publishing NetReady anyway"); - if let Err(e) = this.publish_net_ready() { - error!("Failed to publish NetReady: {e}"); - } - this.net_ready = true; - this.maybe_rebroadcast_own_artifacts(ctx); - } - }); - } - ReadinessDecision::Idle => {} - } - } - - pub fn setup( - bus: &BusHandle, - tx: &mpsc::Sender, - rx: &Arc>, - eventstore: Recipient>, - topic: &str, - ) -> Addr { - let mut events = rx.resubscribe(); - let addr = Self::new(bus, tx, rx, eventstore, topic).start(); - - bus.subscribe(EventType::HistoricalNetSyncStart, addr.clone().recipient()); - - // Forward from NetEvent - tokio::spawn({ - debug!("Spawning event receive loop!"); - let addr = addr.clone(); - async move { - while let Some(event) = super::recv_net_event(&mut events, "NetSyncManager").await { - debug!("Received event {:?}", event); - let delivery = match event { - // Someone is asking for our sync - NetEvent::IncomingRequest(value) => addr.send(value).await, - NetEvent::AllPeersDialed { connected, total } => { - addr.send(AllPeersDialed { connected, total }).await - } - NetEvent::ConnectionEstablished { .. } => addr.send(PeerConnected).await, - _ => continue, - }; - if let Err(error) = delivery { - warn!(%error, "NetSyncManager stopped; ending sync ingress"); - break; - } - } - } - }); - - addr - } -} - -impl Actor for NetSyncManager { - type Context = actix::Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT) - } -} - -/// Event broadcast from event bus -impl Handler for NetSyncManager { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - // We are making a sync request of another node - if let InterfoldEventData::HistoricalNetSyncStart(data) = msg { - // Capture the snapshot-cursor map so we can bound the post-restart re-broadcast of our - // own forwardable artifacts to the in-flight window (H3/H11). - self.rebroadcast_since = Some(data.since.clone().into_iter().collect()); - self.maybe_rebroadcast_own_artifacts(ctx); - ctx.notify(TypedEvent::new(data, ec)) - } - } -} - -/// SyncRequest is called on start up to fetch remote events -impl Handler> for NetSyncManager { - type Result = ResponseFuture<()>; - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - info!("HISTORICAL_NET_SYNC_START"); - trap_fut( - EType::Net, - &self.bus.with_ec(msg.get_ctx()), - handle_sync_request_event( - self.tx.clone(), - self.rx.clone(), - msg, - ctx.address(), - !self.readiness_all_peers_dialed(), - ), - ) - } -} - -impl NetSyncManager { - fn readiness_all_peers_dialed(&self) -> bool { - // `handle_sync_request_event` waits for a connection only if we have not yet observed the - // AllPeersDialed signal. The readiness machine tracks this; mirror its view here. - self.readiness.all_peers_dialed() - } -} - -/// We have received the sync response from the remote peer -impl Handler> for NetSyncManager { - type Result = (); - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - trap(EType::Net, &self.bus.with_ec(msg.get_ctx()), || { - info!("SYNC REQUEST SUCCEEDED"); - let (msg, ctx) = msg.into_components(); - let response = msg.response; - self.bus.publish_from_remote_as_response( - HistoricalNetSyncEventsReceived { - events: response.events.to_vec(), - }, - response.ts, - ctx, - None, - EventSource::Net, - )?; - - Ok(()) - }); - } -} - -/// We have received a sync request from a remote peer -impl Handler for NetSyncManager { - type Result = (); - fn handle(&mut self, msg: IncomingRequest, ctx: &mut Self::Context) -> Self::Result { - trap(EType::Net, &self.bus.clone(), || { - let IncomingRequest { peer, responder } = msg; - let fetch_request: FetchEventsSince = match responder.try_request_into() { - Ok(request) => request, - Err(error) => { - warn!(%peer, %error, "Rejecting malformed historical-sync request"); - responder.bad_request("malformed historical sync request")?; - return Ok(()); - } - }; - if fetch_request.limit() == 0 { - responder.bad_request("limit must be greater than 0")?; - return Ok(()); - } - if let Some(reason) = self.request_capacity_error(&peer) { - warn!( - %peer, - in_flight = self.requests.len(), - "Rejecting historical-sync request: {reason}" - ); - responder.bad_request(reason)?; - return Ok(()); - } - - let id = CorrelationId::new(); - let scan_limit = sync_scan_limit(fetch_request.limit()); - info!( - peer = %peer, - correlation_id = %id, - requested_limit = fetch_request.limit(), - scan_limit, - "Processing incoming historical-sync request" - ); - let query: HashMap = - HashMap::from([(fetch_request.aggregate_id(), fetch_request.since())]); - self.requests - .insert(id, PendingSyncRequest { peer, responder }); - let storage_query = - EventStoreQueryBy::::new(id, query, ctx.address().recipient()) - .with_limit(scan_limit as u64); - if let Err(error) = self.eventstore.try_send(storage_query) { - if let Some(pending) = self.requests.remove(&id) { - pending.responder.respond(ProtocolResponse::Error( - "historical sync storage unavailable".to_string(), - ))?; - } - warn!(%peer, correlation_id = %id, %error, "Failed to query EventStore for sync"); - return Ok(()); - } - ctx.run_later(INCOMING_SYNC_REQUEST_TIMEOUT, move |this, _| { - this.expire_sync_request(id); - }); - Ok(()) - }); - } -} - -/// Receive Events from EventStore -impl Handler for NetSyncManager { - type Result = (); - fn handle(&mut self, msg: EventStoreQueryResponse, _: &mut Self::Context) -> Self::Result { - // Post-restart re-broadcast response (own forwardable artifacts) — handled separately from - // peer sync-request responses. - if self.rebroadcast_query_ids.remove(&msg.id()) { - self.handle_rebroadcast_response(msg.into_events()); - return; - } - trap(EType::Net, &self.bus.clone(), || { - info!("Received response from eventstore."); - let Some(pending) = self.requests.remove(&msg.id()) else { - bail!("responder not found for {}", msg.id()); - }; - - let fetch_request: FetchEventsSince = pending.responder.try_request_into()?; - match build_sync_batch(msg.into_events(), &fetch_request) { - SyncBatchOutcome::BadRequest(reason) => pending.responder.bad_request(reason)?, - SyncBatchOutcome::Batch(batch) => pending.responder.ok(batch)?, - } - - Ok(()) - }) - } -} - -impl Handler for NetSyncManager { - type Result = (); - fn handle(&mut self, msg: AllPeersDialed, ctx: &mut Self::Context) -> Self::Result { - info!( - "NetSyncManager: AllPeersDialed (connected={}, total={})", - msg.connected, msg.total - ); - let decision = self.readiness.on_all_peers_dialed(msg.connected, msg.total); - self.apply_readiness(decision, ctx); - } -} - -impl Handler for NetSyncManager { - type Result = (); - fn handle(&mut self, _: PeerConnected, ctx: &mut Self::Context) -> Self::Result { - let decision = self.readiness.on_peer_connected(); - if let ReadinessDecision::PublishReady = decision { - info!("NetSyncManager: first peer connected"); - } - self.apply_readiness(decision, ctx); - } -} - -#[derive(Message)] -#[rtype(result = "()")] -struct AllPeersDialed { - connected: usize, - total: usize, -} - -#[derive(Message)] -#[rtype(result = "()")] -struct PeerConnected; - -async fn fetch_historical_events_for_aggregate( - net_cmds: &mpsc::Sender, - net_events: &Arc>, - aggregate_id: AggregateId, - since: u128, - budget: &mut SyncFetchBudget, -) -> Result>> { - let requester = DirectRequester::builder(net_cmds.clone(), net_events.clone()) - .max_retries(SYNC_FETCH_MAX_RETRIES) - .retry_timeout(SYNC_FETCH_RETRY_TIMEOUT) - .build(); - - let events = fetch_all_batched_events_with_budget::>( - requester, - PeerTarget::Random, - aggregate_id, - since, - 100, - budget, - ) - .await?; - - validate_historical_events(aggregate_id, events) -} - -fn validate_historical_events( - aggregate_id: AggregateId, - events: Vec>, -) -> Result>> { - for event in &events { - if event.aggregate_id() != aggregate_id { - bail!( - "historical sync peer returned event for aggregate {} while fetching {}", - event.aggregate_id(), - aggregate_id - ); - } - if !EventTranslationService::is_forwardable_event(event) { - bail!( - "historical sync peer returned non-forwardable event type {}", - event.event_type() - ); - } - } - Ok(events) -} - -async fn handle_sync_request_event( - net_cmds: mpsc::Sender, - net_events: Arc>, - event: TypedEvent, - address: impl Into>>, - wait_for_event: bool, -) -> Result<()> { - info!("Sync request event received"); - let (event, ctx) = event.into_components(); - info!("Checking for AllPeersDialed..."); - if wait_for_event { - info!("Waiting for peer connection..."); - let has_peers = await_event( - &net_events, - |e| match e { - NetEvent::ConnectionEstablished { .. } => { - info!("Peer connection established"); - Some(true) - } - NetEvent::AllPeersDialed { total: 0, .. } => { - info!("No peers configured, proceeding without sync"); - Some(false) - } - _ => None, - }, - NET_READY_CONNECT_TIMEOUT, - ) - .await - .context("No peer connections established within timeout")?; - - if !has_peers { - let value = SyncRequestSucceeded { - response: SyncResponseValue { - events: vec![], - ts: 0, - }, - }; - - address.into().try_send(TypedEvent::new(value, ctx))?; - return Ok(()); - } - } - info!("handle_sync_request_event: ready to sync"); - - let mut all_events: Vec> = Vec::new(); - let mut latest_timestamp: u128 = 0; - let mut failed_aggregates: Vec = Vec::new(); - let mut budget = SyncFetchBudget::production(); - - for (aggregate_id, since) in event.since.iter() { - info!( - "Requesting batched events for aggregate_id={} since={}", - aggregate_id, since - ); - match fetch_historical_events_for_aggregate( - &net_cmds, - &net_events, - *aggregate_id, - *since, - &mut budget, - ) - .await - { - Ok(events) => { - info!( - "Received {} events for aggregate_id={}", - events.len(), - aggregate_id - ); - for interfold_event in events { - let ts = interfold_event.ts(); - if ts > latest_timestamp { - latest_timestamp = ts; - } - all_events.push(interfold_event); - } - } - Err(e) => { - if budget.is_exhausted() { - return Err(e).context("historical net sync exhausted its global budget"); - } - warn!( - "Failed to fetch events for aggregate_id={}: {e}. Continuing with available events.", - aggregate_id - ); - failed_aggregates.push(*aggregate_id); - } - } - } - - // If any aggregate failed, retry a few recovery rounds. Prefer a fresh - // ConnectionEstablished signal when one arrives, but do not depend on it: - // a connected peer may simply be slow or temporarily stalled. - if !failed_aggregates.is_empty() { - info!( - "Sync fetch failed for {} aggregates — starting recovery retries...", - failed_aggregates.len() - ); - let mut recovery_attempt = 0; - - while !failed_aggregates.is_empty() && recovery_attempt < SYNC_RECOVERY_MAX_ATTEMPTS { - recovery_attempt += 1; - - match await_event( - &net_events, - |e| { - if matches!(e, NetEvent::ConnectionEstablished { .. }) { - Some(()) - } else { - None - } - }, - SYNC_RECOVERY_RETRY_INTERVAL, - ) - .await - { - Ok(()) => { - info!( - attempt = recovery_attempt, - "Peer reconnected, retrying failed aggregates" - ); - } - Err(_) => { - info!( - attempt = recovery_attempt, - retry_after = ?SYNC_RECOVERY_RETRY_INTERVAL, - "No new peer connection observed; retrying failed aggregates against current peers" - ); - } - } - - let mut still_failed = Vec::new(); - for aggregate_id in failed_aggregates { - let since = event.since.get(&aggregate_id).copied().unwrap_or(0); - match fetch_historical_events_for_aggregate( - &net_cmds, - &net_events, - aggregate_id, - since, - &mut budget, - ) - .await - { - Ok(events) => { - info!( - attempt = recovery_attempt, - "Retry succeeded: {} events for aggregate_id={}", - events.len(), - aggregate_id - ); - for interfold_event in events { - let ts = interfold_event.ts(); - if ts > latest_timestamp { - latest_timestamp = ts; - } - all_events.push(interfold_event); - } - } - Err(e) => { - if budget.is_exhausted() { - return Err(e) - .context("historical net sync exhausted its global budget"); - } - warn!( - attempt = recovery_attempt, - "Retry failed for aggregate_id={}: {e}", aggregate_id - ); - still_failed.push(aggregate_id); - } - } - } - - failed_aggregates = still_failed; - } - - if !failed_aggregates.is_empty() { - bail!( - "failed to fetch historical net events for aggregates: {:?} after {} recovery attempts", - failed_aggregates, - SYNC_RECOVERY_MAX_ATTEMPTS - ); - } - } - - info!( - "Sync complete: collected {} events across {} aggregates, latest_timestamp={}", - all_events.len(), - event.since.len(), - latest_timestamp - ); - - let value = SyncRequestSucceeded { - response: SyncResponseValue { - events: all_events, - ts: latest_timestamp, - }, - }; - - address.into().try_send(TypedEvent::new(value, ctx))?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::{ - direct_responder::ChannelType, - events::{IncomingRequest, NetCommand}, - }; - use actix::{Actor, Context as ActixContext, Handler}; - use e3_ciphernode_builder::EventSystem; - use e3_events::{ - E3id, EventSource, InterfoldEvent, PlaintextAggregated, TestEvent, Unsequenced, - }; - use e3_utils::ArcBytes; - use tokio::sync::{broadcast, mpsc, mpsc::UnboundedSender}; - - /// Minimal EventStore stand-in so `NetSyncManager::new` can be constructed in tests; the - /// re-broadcast unit test drives `handle_rebroadcast_response` directly and never queries it. - struct NoopEventStore; - impl Actor for NoopEventStore { - type Context = ActixContext; - } - impl Handler> for NoopEventStore { - type Result = (); - fn handle(&mut self, _: EventStoreQueryBy, _: &mut Self::Context) {} - } - - struct RecordingEventStore { - queries: UnboundedSender>, - } - - impl Actor for RecordingEventStore { - type Context = ActixContext; - } - - impl Handler> for RecordingEventStore { - type Result = (); - fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) { - let _ = self.queries.send(msg.limit()); - // Intentionally retain no response. Tests exercise the manager's in-flight bounds. - } - } - - fn manager_with_recording_store( - query_tx: UnboundedSender>, - ) -> (NetSyncManager, mpsc::Receiver) { - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle().unwrap().enable("test"); - let (tx, rx) = mpsc::channel::(100); - let (_evt_tx, evt_rx) = broadcast::channel::(100); - let evt_rx = Arc::new(evt_rx); - let eventstore = RecordingEventStore { queries: query_tx } - .start() - .recipient(); - - ( - NetSyncManager::new(&bus, &tx, &evt_rx, eventstore, "my-topic"), - rx, - ) - } - - fn incoming_sync_request( - peer: PeerId, - id: u64, - limit: usize, - tx: &mpsc::Sender, - ) -> IncomingRequest { - let request: Vec = FetchEventsSince::new(AggregateId::new(1), 0, limit) - .try_into() - .unwrap(); - let responder = DirectResponder::new(id, ChannelType::Test(format!("request-{id}")), tx) - .with_request(request); - IncomingRequest { peer, responder } - } - - fn protocol_response(command: NetCommand) -> ProtocolResponse { - let NetCommand::IncomingResponse(incoming) = command else { - panic!("expected IncomingResponse, got {command:?}"); - }; - incoming.responder.to_response().unwrap().1 - } - - fn local_forwardable_event(e3: &str) -> InterfoldEvent { - InterfoldEvent::::new_with_timestamp( - PlaintextAggregated { - e3_id: E3id::new(e3, 1), - decrypted_output: vec![ArcBytes::from_bytes(&[1, 2, 3, 4])], - decryption_aggregator_proofs: vec![], - } - .into(), - None, - 10, - None, - EventSource::Local, - ) - .into_sequenced(1) - } - - fn local_non_forwardable_event() -> InterfoldEvent { - InterfoldEvent::::new_with_timestamp( - TestEvent::new("not-forwardable", 1).into(), - None, - 11, - None, - EventSource::Local, - ) - .into_sequenced(2) - } - - fn remote_unsequenced(event: InterfoldEvent) -> InterfoldEvent { - event.clone_unsequenced().with_source(EventSource::Net) - } - - #[test] - fn historical_sync_rejects_non_forwardable_remote_events() { - let error = validate_historical_events( - AggregateId::new(0), - vec![remote_unsequenced(local_non_forwardable_event())], - ) - .unwrap_err(); - - assert!(error - .to_string() - .contains("non-forwardable event type TestEvent")); - } - - #[test] - fn historical_sync_rejects_events_from_another_aggregate() { - let event = remote_unsequenced(local_forwardable_event("1234")); - - let error = validate_historical_events(AggregateId::new(999), vec![event]).unwrap_err(); - - assert!(error.to_string().contains("while fetching 999")); - } - - #[actix::test] - async fn rebroadcast_only_gossips_forwardable_own_artifacts() { - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle().unwrap().enable("test"); - let (tx, mut rx) = mpsc::channel::(100); - let (_evt_tx, evt_rx) = broadcast::channel::(100); - let evt_rx = Arc::new(evt_rx); - let eventstore = NoopEventStore.start().recipient(); - - let mut mgr = NetSyncManager::new(&bus, &tx, &evt_rx, eventstore, "my-topic"); - - mgr.handle_rebroadcast_response(vec![ - local_forwardable_event("1234"), - local_non_forwardable_event(), - ]); - - // Exactly one GossipPublish for the forwardable artifact, on the configured topic. - let cmd = rx.try_recv().expect("expected a GossipPublish command"); - let NetCommand::GossipPublish { topic, data, .. } = cmd else { - panic!("expected GossipPublish, got {cmd:?}"); - }; - assert_eq!(topic, "my-topic"); - let event: InterfoldEvent = data.try_into().unwrap(); - assert!(matches!( - event.get_data(), - InterfoldEventData::PlaintextAggregated(_) - )); - - // The non-forwardable event must not have produced a second command. - assert!( - rx.try_recv().is_err(), - "non-forwardable event should not be re-broadcast" - ); - } - - #[actix::test] - async fn malicious_huge_limit_is_capped_before_storage_query() { - let (query_tx, mut query_rx) = mpsc::unbounded_channel(); - let (manager, _net_rx) = manager_with_recording_store(query_tx); - let net_tx = manager.tx.clone(); - let manager = manager.start(); - - manager - .send(incoming_sync_request( - PeerId::random(), - 1, - usize::MAX, - &net_tx, - )) - .await - .unwrap(); - - let queried_limit = tokio::time::timeout(Duration::from_secs(1), query_rx.recv()) - .await - .unwrap() - .unwrap(); - assert_eq!(queried_limit, Some(sync_scan_limit(usize::MAX) as u64)); - } - - #[actix::test] - async fn concurrent_sync_requests_are_globally_bounded() { - let (query_tx, mut query_rx) = mpsc::unbounded_channel(); - let (manager, mut net_rx) = manager_with_recording_store(query_tx); - let net_tx = manager.tx.clone(); - let manager = manager.start(); - - for id in 0..MAX_IN_FLIGHT_SYNC_REQUESTS as u64 { - manager - .send(incoming_sync_request(PeerId::random(), id, 1, &net_tx)) - .await - .unwrap(); - } - manager - .send(incoming_sync_request( - PeerId::random(), - MAX_IN_FLIGHT_SYNC_REQUESTS as u64, - 1, - &net_tx, - )) - .await - .unwrap(); - - for _ in 0..MAX_IN_FLIGHT_SYNC_REQUESTS { - tokio::time::timeout(Duration::from_secs(1), query_rx.recv()) - .await - .unwrap() - .unwrap(); - } - assert!( - query_rx.try_recv().is_err(), - "overflow request reached storage" - ); - let response = tokio::time::timeout(Duration::from_secs(1), net_rx.recv()) - .await - .unwrap() - .unwrap(); - assert!(matches!( - protocol_response(response), - ProtocolResponse::BadRequest(reason) if reason.contains("too many in-flight") - )); - } - - #[actix::test] - async fn concurrent_sync_requests_are_bounded_per_authenticated_peer() { - let (query_tx, mut query_rx) = mpsc::unbounded_channel(); - let (manager, mut net_rx) = manager_with_recording_store(query_tx); - let net_tx = manager.tx.clone(); - let manager = manager.start(); - let peer = PeerId::random(); - - for id in 0..=MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER as u64 { - manager - .send(incoming_sync_request(peer, id, 1, &net_tx)) - .await - .unwrap(); - } - - for _ in 0..MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER { - tokio::time::timeout(Duration::from_secs(1), query_rx.recv()) - .await - .unwrap() - .unwrap(); - } - assert!( - query_rx.try_recv().is_err(), - "per-peer overflow reached storage" - ); - let response = tokio::time::timeout(Duration::from_secs(1), net_rx.recv()) - .await - .unwrap() - .unwrap(); - assert!(matches!( - protocol_response(response), - ProtocolResponse::BadRequest(reason) if reason.contains("this peer") - )); - } - - #[actix::test] - async fn timed_out_sync_request_releases_its_in_flight_slot() { - let (query_tx, _query_rx) = mpsc::unbounded_channel(); - let (mut manager, mut net_rx) = manager_with_recording_store(query_tx); - let net_tx = manager.tx.clone(); - let peer = PeerId::random(); - let IncomingRequest { responder, .. } = incoming_sync_request(peer, 1, 1, &net_tx); - let id = CorrelationId::new(); - manager - .requests - .insert(id, PendingSyncRequest { peer, responder }); - - manager.expire_sync_request(id); - - assert!(manager.requests.is_empty()); - let response = tokio::time::timeout(Duration::from_secs(1), net_rx.recv()) - .await - .unwrap() - .unwrap(); - assert!(matches!( - protocol_response(response), - ProtocolResponse::Error(reason) if reason.contains("timed out") - )); - } -} diff --git a/crates/net/src/document_publishing/actor.rs b/crates/net/src/document_publishing/actor.rs new file mode 100644 index 0000000000..9e7daa27d3 --- /dev/null +++ b/crates/net/src/document_publishing/actor.rs @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::{ + domain::{datetime_to_instant_from_now, DocumentPublishingService}, + events::{ + call_and_await_response, DocumentPublishedNotification, GossipData, NetCommand, NetEvent, + }, + ContentHash, +}; +use actix::prelude::*; +use anyhow::{Context, Result}; +use e3_events::{ + prelude::*, trap, trap_fut, BusHandle, CiphernodeSelected, CorrelationId, DocumentReceived, + E3RequestComplete, E3id, EType, EventSource, EventType, InterfoldEvent, InterfoldEventData, + PartyId, PublishDocumentRequested, TypedEvent, +}; +use e3_utils::ArcBytes; +use e3_utils::NotifySync; +use e3_utils::{ + retry::{retry_with_backoff, to_retry}, + MAILBOX_LIMIT, +}; +use futures::TryFutureExt; +use std::{collections::HashMap, sync::Arc, time::Duration}; +use tokio::sync::{broadcast, mpsc}; +use tracing::{debug, info}; + +use super::event_converter::EventConverter; + +const KADEMLIA_PUT_TIMEOUT: Duration = Duration::from_secs(30); +const KADEMLIA_GET_TIMEOUT: Duration = Duration::from_secs(30); +const KADEMLIA_BROADCAST_TIMEOUT: Duration = Duration::from_secs(30); + +/// DocumentPublisher is an actor that monitors events from both the Libp2pNetInterface and the +/// Interfold EventBus in order to manage document publishing interactions. The decision/state logic +/// lives in [`DocumentPublishingService`]; this actor only wires events to that service and +/// performs the resulting libp2p/Kademlia I/O. +pub struct DocumentPublisher { + /// Interfold EventBus + bus: BusHandle, + /// NetCommand sender to forward commands to the Libp2pNetInterface + tx: mpsc::Sender, + /// NetEvent receiver to resubscribe for events from the Libp2pNetInterface. This is in an Arc + /// so that we do not do excessive resubscribes without actually listening for events. + rx: Arc>, + /// The gossipsub broadcast topic + topic: String, + /// Pure decision/state service. + service: DocumentPublishingService, +} + +impl DocumentPublisher { + /// Create a new DocumentPublisher actor + pub fn new( + bus: &BusHandle, + tx: &mpsc::Sender, + rx: &Arc>, + topic: impl Into, + ) -> Self { + Self { + bus: bus.clone(), + tx: tx.clone(), + rx: rx.clone(), + topic: topic.into(), + service: DocumentPublishingService::new(), + } + } + + /// This is needed to create simulation libp2p event routers + pub fn is_document_publisher_event(event: &InterfoldEvent) -> bool { + // Add a list of events with paylods for the DHT + matches!( + event.get_data(), + InterfoldEventData::PublishDocumentRequested(_) + | InterfoldEventData::ThresholdShareCreated(_) + | InterfoldEventData::EncryptionKeyCreated(_) + | InterfoldEventData::DecryptionKeyShared(_) + ) + } + + /// Setup the DocumentPublisher and start listening for GossipEvents + pub fn setup( + bus: &BusHandle, + tx: &mpsc::Sender, + rx: &Arc>, + topic: impl Into, + ) -> Addr { + let mut events = rx.resubscribe(); + let addr = Self::new(bus, tx, rx, topic).start(); + EventConverter::setup(bus); + // Listen on all events + bus.subscribe(EventType::All, addr.clone().recipient()); + + // Forward gossip data from NetEvent + tokio::spawn({ + debug!("Spawning event receive loop!"); + let addr = addr.clone(); + async move { + while let Some(event) = + crate::event_subscription::recv_net_event(&mut events, "DocumentPublisher") + .await + { + debug!("Received event {:?}", event); + if let NetEvent::GossipData(GossipData::DocumentPublishedNotification(data)) = + event + { + if let Err(error) = addr.send(data).await { + tracing::warn!( + %error, + "DocumentPublisher stopped; ending DHT notification ingress" + ); + break; + } + } + } + } + }); + + addr + } + + fn handle_ciphernode_selected(&mut self, event: CiphernodeSelected) -> Result<()> { + let CiphernodeSelected { + e3_id, party_id, .. + } = event; + self.service.register_interest(e3_id, party_id); + Ok(()) + } + + fn handle_e3_request_complete(&mut self, event: E3RequestComplete) -> Result<()> { + let keys = self.service.complete_e3(&event.e3_id); + if !keys.is_empty() { + info!( + "Pruning {} DHT records for completed E3 {}", + keys.len(), + event.e3_id + ); + let _ = self.tx.try_send(NetCommand::DhtRemoveRecords { keys }); + } + Ok(()) + } +} + +#[path = "effects.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +pub use effects::{handle_document_published_notification, handle_publish_document_requested}; + +#[cfg(test)] +#[path = "tests/mod.rs"] +mod tests; diff --git a/crates/net/src/document_publishing/effects.rs b/crates/net/src/document_publishing/effects.rs new file mode 100644 index 0000000000..454ba27370 --- /dev/null +++ b/crates/net/src/document_publishing/effects.rs @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Bounded DHT and gossip effects for content-addressed protocol documents. + +use super::*; +use crate::domain::EventConversionService; + +/// Called when we receive a PublishDocumentRequested event +pub async fn handle_publish_document_requested( + tx: mpsc::Sender, + rx: Arc>, + event: PublishDocumentRequested, + topic: impl Into, + bus: BusHandle, +) -> Result<()> { + let value = event.value; + let key = ContentHash::from_content(&value); + let expires = Some( + datetime_to_instant_from_now(event.meta.expires_at) + .context("refusing to publish an expired DHT document")?, + ); + + retry_with_backoff( + || { + put_record(tx.clone(), rx.clone(), expires, value.clone(), key.clone()) + .map_err(to_retry) + }, + 4, + 1000, + ) + .await?; + let notification = DocumentPublishedNotification::new(event.meta, key, bus.ts()?); + broadcast_document_published_notification(tx, rx, notification, topic).await?; + Ok(()) +} + +/// Called when we receive a notification from the net_interface +pub async fn handle_document_published_notification( + net_cmds: mpsc::Sender, + net_events: Arc>, + bus: BusHandle, + ids: HashMap, + event: DocumentPublishedNotification, +) -> Result<()> { + let Some(party_id) = DocumentPublishingService::interest_in(&ids, &event) else { + debug!("Node not interested in id {}", event.meta.e3_id); + return Ok(()); + }; + + debug!( + "interested in document {:?} with party_id={:?}", + event, party_id + ); + + let value = retry_with_backoff( + || get_record(net_cmds.clone(), net_events.clone(), event.key.clone()).map_err(to_retry), + 4, + 1000, + ) + .await?; + + // The gossiped metadata is not covered by the DHT content hash. Bind it to the decoded + // payload before persisting DocumentReceived; otherwise a notification for an E3 this node is + // interested in can inject a content-addressed document for a different E3 or party route. + EventConversionService::validate_received(&event.meta, &value)?; + + debug!("Sending received event..."); + bus.publish_from_remote( + DocumentReceived { + meta: event.meta, + value, + }, + event.ts, + None, + EventSource::Net, + )?; + + Ok(()) +} + +/// Call DhtPutRecord Command on the Libp2pNetInterface and handle the results +async fn put_record( + net_cmds: mpsc::Sender, + net_events: Arc>, + expires: Option, + value: ArcBytes, + key: ContentHash, +) -> Result<()> { + let id = CorrelationId::new(); + call_and_await_response( + net_cmds, + net_events, + NetCommand::DhtPutRecord { + correlation_id: id, + expires, + value, + key, + }, + |event| match event { + NetEvent::DhtPutRecordSucceeded { .. } => Some(Ok(())), + NetEvent::DhtPutRecordError { error, .. } => { + Some(Err(anyhow::anyhow!("DHT put record failed: {:?}", error))) + } + _ => None, + }, + KADEMLIA_PUT_TIMEOUT, + ) + .await +} + +/// Call DhtGetRecord Command on the Libp2pNetInterface and handle the results +async fn get_record( + net_cmds: mpsc::Sender, + net_events: Arc>, + key: ContentHash, +) -> Result { + let id = CorrelationId::new(); + call_and_await_response( + net_cmds, + net_events, + NetCommand::DhtGetRecord { + correlation_id: id, + key, + }, + |event| match event { + NetEvent::DhtGetRecordSucceeded { value, .. } => Some(Ok(value.clone())), + NetEvent::DhtGetRecordError { error, .. } => { + Some(Err(anyhow::anyhow!("DHT get record failed: {:?}", error))) + } + _ => None, + }, + KADEMLIA_GET_TIMEOUT, + ) + .await +} + +/// Broadcasts document published notification on Libp2pNetInterface +async fn broadcast_document_published_notification( + net_cmds: mpsc::Sender, + net_events: Arc>, + payload: DocumentPublishedNotification, + topic: impl Into, +) -> Result<()> { + let id = CorrelationId::new(); + call_and_await_response( + net_cmds, + net_events, + NetCommand::GossipPublish { + topic: topic.into(), + correlation_id: id, + data: GossipData::DocumentPublishedNotification(payload), + }, + |event| match event { + NetEvent::GossipPublished { .. } => Some(Ok(())), + NetEvent::GossipPublishError { error, .. } => { + Some(Err(anyhow::anyhow!("GossipPublished failed: {:?}", error))) + } + _ => None, + }, + KADEMLIA_BROADCAST_TIMEOUT, + ) + .await +} diff --git a/crates/net/src/document_publishing/handlers.rs b/crates/net/src/document_publishing/handlers.rs new file mode 100644 index 0000000000..184f41fafd --- /dev/null +++ b/crates/net/src/document_publishing/handlers.rs @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Actix routing for document lifecycle and network notifications. + +use super::*; + +impl Actor for DocumentPublisher { + type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } +} + +impl Handler for DocumentPublisher { + type Result = (); + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + match msg { + InterfoldEventData::PublishDocumentRequested(data) => { + ctx.notify(TypedEvent::new(data, ec)) + } + InterfoldEventData::CiphernodeSelected(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::E3RequestComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + _ => (), + } + } +} + +impl Handler> for DocumentPublisher { + type Result = ResponseFuture<()>; + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let tx = self.tx.clone(); + let (msg, ec) = msg.into_components(); + + self.service + .track_published_key(&msg.meta.e3_id, &msg.value); + + let rx = self.rx.clone(); + let bus = self.bus.clone(); + let topic = self.topic.clone(); + trap_fut( + EType::IO, + &bus.with_ec(&ec), + handle_publish_document_requested(tx, rx, msg, topic, bus), + ) + } +} + +impl Handler> for DocumentPublisher { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::DocumentPublishing, &self.bus.with_ec(&ec), || { + self.handle_ciphernode_selected(msg) + }) + } +} + +impl Handler> for DocumentPublisher { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::DocumentPublishing, &self.bus.with_ec(&ec), || { + self.handle_e3_request_complete(msg) + }) + } +} + +/// Receiving DocumentPublishedNotification from libp2p +impl Handler for DocumentPublisher { + type Result = ResponseFuture<()>; + fn handle( + &mut self, + msg: DocumentPublishedNotification, + _: &mut Self::Context, + ) -> Self::Result { + let ids = self.service.interest_snapshot(); + let bus = self.bus.clone(); + let tx = self.tx.clone(); + let rx = self.rx.clone(); + trap_fut( + EType::IO, + &bus, + handle_document_published_notification(tx, rx, bus.clone(), ids, msg), + ) + } +} diff --git a/crates/net/src/document_publishing/tests/mod.rs b/crates/net/src/document_publishing/tests/mod.rs new file mode 100644 index 0000000000..ab3083dcc6 --- /dev/null +++ b/crates/net/src/document_publishing/tests/mod.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::{collections::HashMap, num::NonZero, sync::Arc, time::Duration}; + +use super::*; +use crate::events::NetCommand; +use crate::{domain::EventConversionService, ContentHash}; +use actix::Addr; +use anyhow::{bail, Result}; +use chrono::Utc; +use e3_ciphernode_builder::EventSystem; +use e3_events::{ + BusHandle, CiphernodeSelected, DocumentKind, DocumentMeta, E3id, EncryptionKey, + EncryptionKeyCreated, GetEvents, HistoryCollector, InterfoldError, InterfoldEvent, + PublishDocumentRequested, TakeEvents, +}; +use e3_utils::ArcBytes; +use libp2p::kad::{GetRecordError, PutRecordError, RecordKey}; +use std::time::Instant; +use tokio::{ + sync::{broadcast, mpsc}, + time::{sleep, timeout}, +}; +use tracing::subscriber::DefaultGuard; + +#[allow(clippy::type_complexity)] +fn setup_test() -> Result<( + DefaultGuard, + BusHandle, + mpsc::Sender, + mpsc::Receiver, + broadcast::Sender, + Arc>, + Addr>, + Addr>, + Addr, +)> { + use tracing_subscriber::{fmt, EnvFilter}; + + let subscriber = fmt() + .with_env_filter(EnvFilter::new("debug")) + .with_test_writer() + .finish(); + + let guard = tracing::subscriber::set_default(subscriber); + + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test"); + let (net_cmd_tx, net_cmd_rx) = mpsc::channel(100); + let (net_evt_tx, net_evt_rx) = broadcast::channel(100); + let net_evt_rx = Arc::new(net_evt_rx); + let history = HistoryCollector::::new().start(); + let error = HistoryCollector::::new().start(); + bus.subscribe(EventType::All, history.clone().recipient()); + bus.subscribe(EventType::InterfoldError, error.clone().recipient()); + let publisher = DocumentPublisher::setup(&bus, &net_cmd_tx, &net_evt_rx, "topic"); + + Ok(( + guard, bus, net_cmd_tx, net_cmd_rx, net_evt_tx, net_evt_rx, history, error, publisher, + )) +} + +mod notifications; +mod publishing; + +fn is_between(instant: Instant, start: Instant, end: Instant) -> bool { + let (min, max) = if start <= end { + (start, end) + } else { + (end, start) + }; + instant >= min && instant <= max +} + +fn days_from_now(days: u64) -> Instant { + Instant::now() + Duration::from_secs(60 * 60 * 24 * days) +} diff --git a/crates/net/src/document_publishing/tests/notifications.rs b/crates/net/src/document_publishing/tests/notifications.rs new file mode 100644 index 0000000000..8efb0ca7e9 --- /dev/null +++ b/crates/net/src/document_publishing/tests/notifications.rs @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn test_notified_of_document() -> Result<()> { + let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, history, _, _) = + setup_test()?; + + let expires_at = Utc::now() + chrono::Duration::days(1); + let e3_id = E3id::new("1243", 1); + let value = EventConversionService::encryption_key_to_request(EncryptionKeyCreated { + e3_id: e3_id.clone(), + key: Arc::new(EncryptionKey::new(1, ArcBytes::from_bytes(b"public key"))), + external: false, + })? + .expect("local key should produce a document") + .value; + let cid = ContentHash::from_content(&value); + + // 1. Ensure the publisher is interested in the id by receiving CiphernodeSelected + bus.publish_without_context(CiphernodeSelected { + e3_id: e3_id.clone(), + threshold_m: 3, + threshold_n: 5, + ..CiphernodeSelected::default() + })?; + + // 2. Dispatch a NetEvent from the Libp2pNetInterface signaling that a document was published + net_evt_tx.send(NetEvent::GossipData( + GossipData::DocumentPublishedNotification(DocumentPublishedNotification { + key: ContentHash::from_content(b"wrong document".as_ref()), + meta: DocumentMeta::new( + E3id::new("1111", 1), + DocumentKind::TrBFV, + vec![], + Some(expires_at), + ), + ts: 123, + }), + ))?; + + // 3. Nothing happens... + let result = timeout(Duration::from_secs(1), net_cmd_rx.recv()).await; + assert!(result.is_err(), "Expected timeout but received a message"); + + // 4. Dispatch a NetEvent from the Libp2pNetInterface signaling that a document we ARE interested + // in was published + net_evt_tx.send(NetEvent::GossipData( + GossipData::DocumentPublishedNotification(DocumentPublishedNotification { + key: cid.clone(), + meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], Some(expires_at)), + ts: 100, + }), + ))?; + + // 5. Expect that DocumentPublisher will make a DhtGetRecord request + let Some(NetCommand::DhtGetRecord { + key, + correlation_id, + }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()) + .await + .expect("did not receive DhtGetRecord") + else { + bail!("msg not as expected"); + }; + + assert_eq!(key, cid); + + // 6. Forward the document + net_evt_tx.send(NetEvent::DhtGetRecordSucceeded { + key: cid, + correlation_id, + value: value.clone(), + })?; + + // wait for events to settle + sleep(Duration::from_millis(100)).await; + + // Check event was dispatched + let events = history.send(GetEvents::new()).await?; + let Some(InterfoldEventData::DocumentReceived(DocumentReceived { value: doc, .. })) = + events.iter().find_map(|event| match event.get_data() { + data @ InterfoldEventData::DocumentReceived(_) => Some(data), + _ => None, + }) + else { + bail!("No event sent"); + }; + + assert_eq!( + doc.extract_bytes(), + value.extract_bytes(), + "document did not match" + ); + + Ok(()) +} + +#[actix::test] +async fn notification_cannot_relabel_payload_for_another_e3() -> Result<()> { + let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _rx, history, errors, _) = + setup_test()?; + let interested_e3 = E3id::new("100", 1); + let payload_e3 = E3id::new("200", 1); + let value = EventConversionService::encryption_key_to_request(EncryptionKeyCreated { + e3_id: payload_e3, + key: Arc::new(EncryptionKey::new(1, ArcBytes::from_bytes(b"public key"))), + external: false, + })? + .expect("local key should produce a document") + .value; + let key = ContentHash::from_content(&value); + + bus.publish_without_context(CiphernodeSelected { + e3_id: interested_e3.clone(), + threshold_m: 3, + threshold_n: 5, + ..CiphernodeSelected::default() + })?; + net_evt_tx.send(NetEvent::GossipData( + GossipData::DocumentPublishedNotification(DocumentPublishedNotification { + key: key.clone(), + meta: DocumentMeta::new( + interested_e3, + DocumentKind::TrBFV, + vec![], + Some(Utc::now() + chrono::Duration::days(1)), + ), + ts: 100, + }), + ))?; + + let Some(NetCommand::DhtGetRecord { correlation_id, .. }) = + timeout(Duration::from_secs(1), net_cmd_rx.recv()) + .await + .expect("did not receive DhtGetRecord") + else { + bail!("msg not as expected"); + }; + net_evt_tx.send(NetEvent::DhtGetRecordSucceeded { + key, + correlation_id, + value, + })?; + + let error_events = errors.send(TakeEvents::new(1)).await?; + let error: InterfoldError = error_events.events.first().unwrap().try_into()?; + assert!(error.message.contains("metadata E3 1:100")); + assert!(error.message.contains("payload E3 1:200")); + + let events = history.send(GetEvents::new()).await?; + assert!( + !events + .iter() + .any(|event| matches!(event.get_data(), InterfoldEventData::DocumentReceived(_))), + "mismatched document must not be persisted" + ); + Ok(()) +} diff --git a/crates/net/src/document_publishing/tests/publishing.rs b/crates/net/src/document_publishing/tests/publishing.rs new file mode 100644 index 0000000000..379d52781f --- /dev/null +++ b/crates/net/src/document_publishing/tests/publishing.rs @@ -0,0 +1,211 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn test_publishes_document() -> Result<()> { + let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _, _, _) = + setup_test()?; + let value = ArcBytes::from_bytes(b"I am a special document"); + let expires_at = Some(Utc::now() + chrono::Duration::days(1)); + let e3_id = E3id::new("1243", 1); + + // 1. Send a request to publish + bus.publish_without_context(PublishDocumentRequested { + meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), + value: value.clone(), + })?; + + // 2. Document publisher should have asked the Libp2pNetInterface to put the doc on Kademlia + let Some(NetCommand::DhtPutRecord { + correlation_id, + expires, + value: msg_value, + key, + }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()) + .await + .expect("did not receive DhtPutRecord") + else { + bail!("msg not as expected"); + }; + + // Fake DHT put the record + let mut mykad: HashMap> = HashMap::new(); + mykad.insert(key.clone(), msg_value.extract_bytes()); + + // 3. Report that everything went well + net_evt_tx.send(NetEvent::DhtPutRecordSucceeded { + correlation_id, + key, + })?; + + // 4. Expect a DocumentPublishedNotification to have been emitted + let Some(NetCommand::GossipPublish { + topic, + correlation_id, + data: GossipData::DocumentPublishedNotification(notification), + .. + }) = timeout(Duration::from_secs(1), net_cmd_rx.recv()) + .await + .expect("did not receive GossipPublish") + else { + bail!("msg not as expected"); + }; + + // 5. Report everything went well + net_evt_tx.send(NetEvent::GossipPublished { + correlation_id, + message_id: libp2p::gossipsub::MessageId::new(&[1, 2, 3]), + })?; + + assert_eq!(topic, "topic"); + assert_eq!(notification.meta.e3_id, E3id::new("1243", 1)); + + assert_eq!( + mykad.get(¬ification.key), + Some(&b"I am a special document".to_vec()), + "value was not correct" + ); + + assert!( + is_between(expires.unwrap(), days_from_now(0), days_from_now(1)), + "Expiry was not set" + ); + + Ok(()) +} + +#[actix::test] +async fn expired_document_is_rejected_without_a_dht_write() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("expired-document"); + let (net_cmd_tx, mut net_cmd_rx) = mpsc::channel(1); + let (_net_evt_tx, net_evt_rx) = broadcast::channel(1); + let event = PublishDocumentRequested { + meta: DocumentMeta::new( + E3id::new("expired", 1), + DocumentKind::TrBFV, + vec![], + Some(Utc::now() - chrono::Duration::seconds(1)), + ), + value: ArcBytes::from_bytes(b"stale"), + }; + + let error = + handle_publish_document_requested(net_cmd_tx, Arc::new(net_evt_rx), event, "topic", bus) + .await + .unwrap_err(); + + assert!(format!("{error:#}").contains("expiry is not in the future")); + assert!(matches!( + net_cmd_rx.try_recv(), + Err(mpsc::error::TryRecvError::Empty | mpsc::error::TryRecvError::Disconnected) + )); + Ok(()) +} + +#[actix::test] +async fn test_get_document_fails_with_exponential_backoff() -> Result<()> { + let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _, errors, _) = + setup_test()?; + + let value = b"I am a special document".to_vec(); + let expires_at = Some(Utc::now() + chrono::Duration::days(1)); + let e3_id = E3id::new("1243", 1); + let cid = ContentHash::from_content(&value); + + // 1. Ensure the publisher is interested in the id by receiving CiphernodeSelected + bus.publish_without_context(CiphernodeSelected { + e3_id: e3_id.clone(), + threshold_m: 3, + threshold_n: 5, + ..CiphernodeSelected::default() + })?; + + net_evt_tx.send(NetEvent::GossipData( + GossipData::DocumentPublishedNotification(DocumentPublishedNotification { + key: cid.clone(), + meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), + ts: 123, + }), + ))?; + + for _ in 0..4 { + // Expect retry + let Some(NetCommand::DhtGetRecord { correlation_id, .. }) = + timeout(Duration::from_secs(15), net_cmd_rx.recv()) + .await + .expect("did not receive DhtGetRecord") + else { + bail!("msg not as expected"); + }; + + // Report failure + net_evt_tx.send(NetEvent::DhtGetRecordError { + correlation_id, + error: GetRecordError::Timeout { + key: RecordKey::new(&cid), + }, + })?; + } + + // wait for events to settle + let errors = errors.send(TakeEvents::new(1)).await?; + let error: InterfoldError = errors.events.first().unwrap().try_into()?; + assert_eq!( + error.message, + "Operation failed after 4 attempts. Last error: DHT get record failed: Timeout { key: Key(b\"\\xda-\\xe1\\xc0T\\x11$X\\x05\\xd1\\xd4\\xa6C\\x86\\x96\\xb7e\\xd9j\\x96\\x1bD\\xc8P#\\x0f\\\"\\xea A@b\") }" + ); + + Ok(()) +} + +#[actix::test] +async fn test_publishes_document_fails_with_exponential_backoff() -> Result<()> { + let (_guard, bus, _net_cmd_tx, mut net_cmd_rx, net_evt_tx, _net_evt_rx, _history, errors, _) = + setup_test()?; + let value = ArcBytes::from_bytes(b"I am a special document"); + let expires_at = Some(Utc::now() + chrono::Duration::days(1)); + let e3_id = E3id::new("1243", 1); + + // Send a request to publish + bus.publish_without_context(PublishDocumentRequested { + meta: DocumentMeta::new(e3_id, DocumentKind::TrBFV, vec![], expires_at), + value: value.clone(), + })?; + + for _ in 0..4 { + // Expect retry + let Some(NetCommand::DhtPutRecord { correlation_id, .. }) = + timeout(Duration::from_secs(15), net_cmd_rx.recv()) + .await + .expect("did not receive DhtPutRecord") + else { + bail!("msg not as expected"); + }; + + // Report failure + net_evt_tx.send(NetEvent::DhtPutRecordError { + correlation_id, + error: crate::events::PutOrStoreError::PutRecordError(PutRecordError::QuorumFailed { + key: RecordKey::new(b"I got the secret"), + success: vec![], + quorum: NonZero::new(1).unwrap(), + }), + })?; + } + + // Expect error to exist + let errors = errors.send(TakeEvents::new(1)).await?; + let error: InterfoldError = errors.events.first().unwrap().try_into()?; + assert_eq!( + error.message, + "Operation failed after 4 attempts. Last error: DHT put record failed: PutRecordError(QuorumFailed { key: Key(b\"I got the secret\"), success: [], quorum: 1 })" + ); + + Ok(()) +} diff --git a/crates/net/src/domain/document_publishing.rs b/crates/net/src/document_publishing/workflow.rs similarity index 100% rename from crates/net/src/domain/document_publishing.rs rename to crates/net/src/document_publishing/workflow.rs diff --git a/crates/net/src/domain/mod.rs b/crates/net/src/domain.rs similarity index 69% rename from crates/net/src/domain/mod.rs rename to crates/net/src/domain.rs index ad90222668..cbde1daf6f 100644 --- a/crates/net/src/domain/mod.rs +++ b/crates/net/src/domain.rs @@ -4,20 +4,30 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -//! Pure, synchronous, unit-testable domain services for the net crate. +//! Compatibility view of pure services stored with their network capabilities. //! //! These contain all decision/state logic that the actix actors and transport layer rely on. //! Nothing here touches actix, the event bus, channels, or libp2p directly. +#[path = "network_sync/correlator.rs"] pub(crate) mod correlator; +#[path = "document_publishing/workflow.rs"] pub(crate) mod document_publishing; +#[path = "event_conversion/workflow.rs"] pub(crate) mod event_conversion; +#[path = "event_translation/workflow.rs"] pub(crate) mod event_translation; +#[path = "event_buffer/workflow.rs"] pub(crate) mod net_buffer; +#[path = "event_buffer/model.rs"] pub(crate) mod net_event_batch; +#[path = "network_status.rs"] mod network_status; +#[path = "peer_failure_tracker.rs"] pub(crate) mod peer_failure_tracker; +#[path = "network_sync/workflow.rs"] pub(crate) mod sync_coordinator; +#[path = "network_sync/wire.rs"] pub(crate) mod wire; pub use document_publishing::{datetime_to_instant_from_now, DocumentPublishingService}; diff --git a/crates/net/src/domain/net_event_batch.rs b/crates/net/src/domain/net_event_batch.rs deleted file mode 100644 index 6dcba5107a..0000000000 --- a/crates/net/src/domain/net_event_batch.rs +++ /dev/null @@ -1,583 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use std::fmt::Debug; - -use std::time::Duration; - -use anyhow::{bail, ensure, Context, Result}; -use e3_events::AggregateId; -use serde::Serialize; -use tokio::time::Instant; -use tracing::info; - -use crate::{ - direct_requester::{DirectRequester, WithPeer, WithoutPeer}, - domain::sync_coordinator::effective_sync_limit, - domain::wire::{decode, MAX_DIRECT_MESSAGE_BYTES}, - events::PeerTarget, -}; - -/// Startup sync is a recovery aid, not an unbounded bulk-transfer protocol. These limits prevent -/// a peer from keeping a node in startup indefinitely or retaining an attacker-controlled history -/// in memory. Operators with a larger legitimate gap must use the controlled resync procedure. -const MAX_SYNC_FETCH_PAGES: usize = 512; -const MAX_SYNC_FETCH_EVENTS: usize = 50_000; -const MAX_SYNC_FETCH_BYTES: usize = 128 * 1024 * 1024; -const MAX_SYNC_FETCH_DURATION: Duration = Duration::from_secs(5 * 60); - -pub(crate) struct SyncFetchBudget { - started: Instant, - pages: usize, - events: usize, - bytes: usize, - max_pages: usize, - max_events: usize, - max_bytes: usize, - max_duration: Duration, - exhausted: bool, -} - -impl SyncFetchBudget { - pub(crate) fn production() -> Self { - Self::new( - MAX_SYNC_FETCH_PAGES, - MAX_SYNC_FETCH_EVENTS, - MAX_SYNC_FETCH_BYTES, - ) - } - - fn new(max_pages: usize, max_events: usize, max_bytes: usize) -> Self { - Self { - started: Instant::now(), - pages: 0, - events: 0, - bytes: 0, - max_pages, - max_events, - max_bytes, - max_duration: MAX_SYNC_FETCH_DURATION, - exhausted: false, - } - } - - pub(crate) fn is_exhausted(&self) -> bool { - self.exhausted - } - - fn remaining(&mut self) -> Result { - match self.max_duration.checked_sub(self.started.elapsed()) { - Some(remaining) => Ok(remaining), - None => { - self.exhausted = true; - bail!("historical sync exceeded total deadline") - } - } - } - - fn record_page(&mut self, event_count: usize, encoded_bytes: usize) -> Result<()> { - let pages = self - .pages - .checked_add(1) - .context("sync page count overflow")?; - let events = self - .events - .checked_add(event_count) - .context("sync event count overflow")?; - let bytes = self - .bytes - .checked_add(encoded_bytes) - .context("sync byte count overflow")?; - - if pages > self.max_pages { - self.exhausted = true; - bail!("historical sync exceeded page limit ({})", self.max_pages); - } - if events > self.max_events { - self.exhausted = true; - bail!("historical sync exceeded event limit ({})", self.max_events); - } - if bytes > self.max_bytes { - self.exhausted = true; - bail!("historical sync exceeded byte limit ({})", self.max_bytes); - } - - self.pages = pages; - self.events = events; - self.bytes = bytes; - Ok(()) - } -} - -#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] -pub enum BatchCursor { - Done, - Next(u128), -} - -#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] -pub struct EventBatch { - pub events: Vec, - pub next: BatchCursor, - pub aggregate_id: AggregateId, -} - -impl TryFrom> for EventBatch -where - E: serde::de::DeserializeOwned, -{ - type Error = anyhow::Error; - - fn try_from(value: Vec) -> Result { - decode(&value, MAX_DIRECT_MESSAGE_BYTES).context("failed to deserialize EventBatch") - } -} - -impl TryFrom> for Vec -where - E: serde::Serialize, -{ - type Error = anyhow::Error; - - fn try_from(value: EventBatch) -> Result { - bincode::serialize(&value).context("failed to serialize EventBatch") - } -} - -#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] -pub struct FetchEventsSince { - aggregate_id: AggregateId, - since: u128, - limit: usize, -} - -impl FetchEventsSince { - pub fn new(aggregate_id: AggregateId, since: u128, limit: usize) -> Self { - Self { - aggregate_id, - since, - limit, - } - } - - pub fn aggregate_id(&self) -> AggregateId { - self.aggregate_id - } - - pub fn since(&self) -> u128 { - self.since - } - - pub fn limit(&self) -> usize { - self.limit - } -} - -impl TryFrom for Vec { - type Error = anyhow::Error; - - fn try_from(value: FetchEventsSince) -> Result { - bincode::serialize(&value).context("failed to serialize FetchEventsSince") - } -} - -impl TryFrom> for FetchEventsSince { - type Error = anyhow::Error; - - fn try_from(value: Vec) -> Result { - decode(&value, MAX_DIRECT_MESSAGE_BYTES).context("failed to deserialize FetchEventsSince") - } -} - -pub async fn fetch_events_since( - requester: &DirectRequester, - request: FetchEventsSince, -) -> Result> -where - E: Debug + TryFrom> + Send + Sync + 'static, - EventBatch: TryFrom>, -{ - requester.request(request).await -} - -#[cfg(test)] -async fn fetch_all_batched_events( - requester: DirectRequester, - peer: PeerTarget, - aggregate_id: AggregateId, - since: u128, - batch_size: usize, -) -> Result> -where - E: Debug + Serialize + TryFrom> + Send + Sync + 'static, - EventBatch: TryFrom>, -{ - let mut budget = SyncFetchBudget::production(); - fetch_all_batched_events_with_budget( - requester, - peer, - aggregate_id, - since, - batch_size, - &mut budget, - ) - .await -} - -pub(crate) async fn fetch_all_batched_events_with_budget( - requester: DirectRequester, - peer: PeerTarget, - aggregate_id: AggregateId, - since: u128, - batch_size: usize, - budget: &mut SyncFetchBudget, -) -> Result> -where - E: Debug + Serialize + TryFrom> + Send + Sync + 'static, - EventBatch: TryFrom>, -{ - ensure!(batch_size > 0, "sync batch size must be greater than 0"); - let batch_size = effective_sync_limit(batch_size); - let requester = requester.to(peer); - let mut all_events = Vec::new(); - let mut cursor = since; - - loop { - let request = FetchEventsSince::new(aggregate_id, cursor, batch_size); - info!( - "Fetching batch aggregate={} cursor={} batch_size={}", - aggregate_id, cursor, batch_size - ); - let remaining = budget.remaining()?; - let batch = - match tokio::time::timeout(remaining, fetch_events_since(&requester, request)).await { - Ok(result) => result?, - Err(_) => { - budget.exhausted = true; - bail!("historical sync exceeded total deadline"); - } - }; - ensure!( - batch.aggregate_id == aggregate_id, - "sync peer returned aggregate {} while fetching {}", - batch.aggregate_id, - aggregate_id - ); - ensure!( - batch.events.len() <= batch_size, - "sync peer returned {} events, exceeding requested batch size {}", - batch.events.len(), - batch_size - ); - info!( - "Batch received with {} events for aggregate={} cursor={}", - batch.events.len(), - aggregate_id, - cursor - ); - - let page_bytes: usize = bincode::serialized_size(&batch.events) - .context("failed to measure historical sync page")? - .try_into() - .context("historical sync page size does not fit usize")?; - budget.record_page(batch.events.len(), page_bytes)?; - all_events - .try_reserve(batch.events.len()) - .map_err(|error| { - anyhow::anyhow!("failed to reserve historical sync buffer: {error}") - })?; - all_events.extend(batch.events); - - match batch.next { - BatchCursor::Done => break, - BatchCursor::Next(next_cursor) => { - ensure!( - next_cursor > cursor, - "sync peer returned non-advancing cursor {next_cursor} from {cursor}" - ); - cursor = next_cursor; - } - } - } - - info!("Batch is done returning {} events", all_events.len()); - - Ok(all_events) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::direct_requester::DirectRequesterTester; - use crate::events::{NetCommand, NetEvent, PeerTarget}; - use std::sync::Arc; - use tokio::sync::{broadcast, mpsc}; - - #[test] - fn sync_fetch_budget_rejects_each_resource_limit_without_mutating_state() { - let mut budget = SyncFetchBudget::new(1, 2, 3); - budget.record_page(2, 3).unwrap(); - - let page_error = budget.record_page(0, 0).unwrap_err(); - assert!(page_error.to_string().contains("page limit")); - assert_eq!((budget.pages, budget.events, budget.bytes), (1, 2, 3)); - - let mut event_budget = SyncFetchBudget::new(2, 1, 3); - let event_error = event_budget.record_page(2, 0).unwrap_err(); - assert!(event_error.to_string().contains("event limit")); - assert_eq!( - (event_budget.pages, event_budget.events, event_budget.bytes), - (0, 0, 0) - ); - - let mut byte_budget = SyncFetchBudget::new(2, 2, 1); - let byte_error = byte_budget.record_page(0, 2).unwrap_err(); - assert!(byte_error.to_string().contains("byte limit")); - assert_eq!( - (byte_budget.pages, byte_budget.events, byte_budget.bytes), - (0, 0, 0) - ); - } - - #[tokio::test] - async fn test_fetch_all_batched_events() { - let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); - let (net_events_tx, net_events_rx) = broadcast::channel::(16); - let net_events = Arc::new(net_events_rx); - - let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); - - let batch1 = EventBatch { - events: vec![b"event1".to_vec(), b"event2".to_vec()], - next: BatchCursor::Next(100), - aggregate_id: AggregateId::new(1), - }; - let batch2 = EventBatch { - events: vec![b"event3".to_vec()], - next: BatchCursor::Done, - aggregate_id: AggregateId::new(1), - }; - - let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) - .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 100)) - .respond_with(batch1) - .expect_request(FetchEventsSince::new(AggregateId::new(1), 100, 100)) - .respond_with(batch2) - .spawn(); - - let events: Vec> = - fetch_all_batched_events(requester, PeerTarget::Random, AggregateId::new(1), 0, 100) - .await - .unwrap(); - - handle.await.unwrap(); - - assert_eq!( - events, - vec![b"event1".to_vec(), b"event2".to_vec(), b"event3".to_vec(),] - ); - } - - #[tokio::test] - async fn test_no_duplicate_events_across_batches() { - let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); - let (net_events_tx, net_events_rx) = broadcast::channel::(16); - let net_events = Arc::new(net_events_rx); - - let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); - - let batch1 = EventBatch { - events: vec![b"event1".to_vec(), b"event2".to_vec()], - next: BatchCursor::Next(2), - aggregate_id: AggregateId::new(1), - }; - let batch2 = EventBatch { - events: vec![b"event3".to_vec()], - next: BatchCursor::Done, - aggregate_id: AggregateId::new(1), - }; - - let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) - .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 100)) - .respond_with(batch1) - .expect_request(FetchEventsSince::new(AggregateId::new(1), 2, 100)) - .respond_with(batch2) - .spawn(); - - let events: Vec> = - fetch_all_batched_events(requester, PeerTarget::Random, AggregateId::new(1), 0, 100) - .await - .unwrap(); - - handle.await.unwrap(); - - let unique: Vec<_> = events - .iter() - .collect::>() - .into_iter() - .collect(); - assert_eq!( - events.len(), - unique.len(), - "Found duplicate events: total={}, unique={}", - events.len(), - unique.len() - ); - assert_eq!( - events, - vec![b"event1".to_vec(), b"event2".to_vec(), b"event3".to_vec()] - ); - } - - #[tokio::test] - async fn test_cursor_correctly_excludes_last_event_on_batch_boundary() { - let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); - let (net_events_tx, net_events_rx) = broadcast::channel::(16); - let net_events = Arc::new(net_events_rx); - - let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); - - let batch1 = EventBatch { - events: vec![b"event1".to_vec()], - next: BatchCursor::Next(100), - aggregate_id: AggregateId::new(1), - }; - let batch2 = EventBatch { - events: vec![b"event2".to_vec()], - next: BatchCursor::Done, - aggregate_id: AggregateId::new(1), - }; - - let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) - .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 1)) - .respond_with(batch1) - .expect_request(FetchEventsSince::new(AggregateId::new(1), 100, 1)) - .respond_with(batch2) - .spawn(); - - let events: Vec> = - fetch_all_batched_events(requester, PeerTarget::Random, AggregateId::new(1), 0, 1) - .await - .unwrap(); - - handle.await.unwrap(); - - assert_eq!(events.len(), 2); - assert_eq!(events[0], b"event1"); - assert_eq!(events[1], b"event2"); - } - - #[tokio::test] - async fn test_non_advancing_cursor_is_rejected() { - let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); - let (net_events_tx, net_events_rx) = broadcast::channel::(16); - let net_events = Arc::new(net_events_rx); - - let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); - let batch = EventBatch { - events: vec![b"event1".to_vec()], - next: BatchCursor::Next(0), - aggregate_id: AggregateId::new(1), - }; - let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) - .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 1)) - .respond_with(batch) - .spawn(); - - let error = fetch_all_batched_events::>( - requester, - PeerTarget::Random, - AggregateId::new(1), - 0, - 1, - ) - .await - .unwrap_err(); - - handle.await.unwrap(); - assert!(error.to_string().contains("non-advancing cursor")); - } - - #[tokio::test] - async fn test_single_batch_with_limit_exactly_matched_returns_done() { - let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); - let (net_events_tx, net_events_rx) = broadcast::channel::(16); - let net_events = Arc::new(net_events_rx); - - let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); - - let batch = EventBatch { - events: vec![b"event1".to_vec()], - next: BatchCursor::Done, - aggregate_id: AggregateId::new(1), - }; - - let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) - .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 1)) - .respond_with(batch) - .spawn(); - - let events: Vec> = - fetch_all_batched_events(requester, PeerTarget::Random, AggregateId::new(1), 0, 1) - .await - .unwrap(); - - handle.await.unwrap(); - - assert_eq!(events, vec![b"event1".to_vec()]); - } - - #[tokio::test] - async fn test_three_batches_with_cursor_continuity() { - let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); - let (net_events_tx, net_events_rx) = broadcast::channel::(16); - let net_events = Arc::new(net_events_rx); - - let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); - - let batch1 = EventBatch { - events: vec![b"a".to_vec(), b"b".to_vec()], - next: BatchCursor::Next(200), - aggregate_id: AggregateId::new(1), - }; - let batch2 = EventBatch { - events: vec![b"c".to_vec(), b"d".to_vec()], - next: BatchCursor::Next(400), - aggregate_id: AggregateId::new(1), - }; - let batch3 = EventBatch { - events: vec![b"e".to_vec()], - next: BatchCursor::Done, - aggregate_id: AggregateId::new(1), - }; - - let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) - .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 2)) - .respond_with(batch1) - .expect_request(FetchEventsSince::new(AggregateId::new(1), 200, 2)) - .respond_with(batch2) - .expect_request(FetchEventsSince::new(AggregateId::new(1), 400, 2)) - .respond_with(batch3) - .spawn(); - - let events: Vec> = - fetch_all_batched_events(requester, PeerTarget::Random, AggregateId::new(1), 0, 2) - .await - .unwrap(); - - handle.await.unwrap(); - - let expected = vec![ - b"a".to_vec(), - b"b".to_vec(), - b"c".to_vec(), - b"d".to_vec(), - b"e".to_vec(), - ]; - assert_eq!(events.len(), expected.len()); - assert_eq!(events, expected); - } -} diff --git a/crates/net/src/event_buffer/actor.rs b/crates/net/src/event_buffer/actor.rs new file mode 100644 index 0000000000..f7318de3b1 --- /dev/null +++ b/crates/net/src/event_buffer/actor.rs @@ -0,0 +1,124 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, ActorContext, AsyncContext, Handler, Message}; +use anyhow::{anyhow, Context, Result}; +use e3_events::{ + BusHandle, EType, ErrorDispatcher, Event, EventSubscriber, EventType, InterfoldEvent, + InterfoldEventData, +}; +use e3_utils::MAILBOX_LIMIT; +use tokio::sync::broadcast::{self, error::RecvError}; +use tokio::sync::oneshot; + +use crate::domain::net_buffer::{BufferDecision, NetEventBufferState}; +use crate::events::NetEvent; + +pub const DEFAULT_MAX_BUFFERED_NET_EVENTS: usize = 1_024; +pub const DEFAULT_MAX_BUFFERED_NET_BYTES: usize = 256 * 1024 * 1024; + +pub struct NetEventBufferHandle { + readiness: oneshot::Receiver>, +} + +impl NetEventBufferHandle { + pub async fn wait_until_running(self) -> Result<()> { + self.readiness + .await + .context("network event buffer stopped before reporting startup status")? + .map_err(anyhow::Error::msg) + } +} + +/// Actor that controls a broadcast channel which will buffer NetEvents until it receives a +/// `SyncEnded` event, at which time it releases all buffered events to the output channel. The +/// buffering decision logic lives in [`NetEventBufferState`]. +pub struct NetEventBuffer { + state: NetEventBufferState, + input_rx: Option>, + output_tx: broadcast::Sender, + bus: BusHandle, + max_events: usize, + max_bytes: usize, + readiness: Option>>, +} + +impl NetEventBuffer { + pub(crate) fn setup_with_limits( + bus: &BusHandle, + input_rx: &broadcast::Receiver, + max_events: usize, + max_bytes: usize, + ) -> (broadcast::Receiver, NetEventBufferHandle) { + let input_rx = input_rx.resubscribe(); + let (output_tx, output_rx) = broadcast::channel(max_events); + let (readiness_tx, readiness) = oneshot::channel(); + + let actor = Self { + state: NetEventBufferState::syncing(), + input_rx: Some(input_rx), + output_tx, + bus: bus.clone(), + max_events, + max_bytes, + readiness: Some(readiness_tx), + }; + + let addr = actor.start(); + + // Subscribe to InterfoldEvent on the bus + bus.subscribe(EventType::SyncEnded, addr.clone().recipient()); + + (output_rx, NetEventBufferHandle { readiness }) + } + + fn handle_interfold_event(&mut self, msg: InterfoldEvent) -> Result<()> { + if let InterfoldEventData::SyncEnded(_) = msg.get_data() { + return self.process_sync_ended(); + } + Ok(()) + } + + fn process_sync_ended(&mut self) -> Result<()> { + let pending = self.state.run()?; + for event in pending { + self.forward_event(event)?; + } + self.signal_startup(Ok(())); + Ok(()) + } + + fn forward_event(&mut self, event: NetEvent) -> Result<()> { + self.output_tx + .send(event) + .map_err(|e| anyhow!("Failed to forward event: {}", e))?; + Ok(()) + } + + fn signal_startup(&mut self, result: std::result::Result<(), String>) { + if let Some(sender) = self.readiness.take() { + let _ = sender.send(result); + } + } + + fn fail_closed(&mut self, error: anyhow::Error, ctx: &mut actix::Context) { + let reason = format!( + "network event buffer failed closed: {error:#}; startup will stop rather than drop \ + live protocol input. Increase the configured buffer only after measuring the sync \ + backlog, or restore peer/RPC health and restart" + ); + self.signal_startup(Err(reason.clone())); + self.bus.err(EType::Net, anyhow!(reason)); + ctx.stop(); + } +} + +#[path = "handlers.rs"] +mod handlers; + +#[cfg(test)] +#[path = "tests.rs"] +mod tests; diff --git a/crates/net/src/event_buffer/handlers.rs b/crates/net/src/event_buffer/handlers.rs new file mode 100644 index 0000000000..db420ba6b1 --- /dev/null +++ b/crates/net/src/event_buffer/handlers.rs @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Mailbox entry points and lifecycle hooks. + +use super::*; + +#[derive(Message)] +#[rtype(result = "()")] +struct IncomingNetEvent(NetEvent); + +#[derive(Message)] +#[rtype(result = "()")] +struct NetInputLagged(u64); + +impl Actor for NetEventBuffer { + type Context = actix::Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + let addr = ctx.address(); + let mut input_rx = self.input_rx.take().expect("input_rx should be present"); + + actix::spawn(async move { + loop { + match input_rx.recv().await { + Ok(event) => { + if addr.send(IncomingNetEvent(event)).await.is_err() { + break; + } + } + Err(RecvError::Lagged(skipped)) => { + let _ = addr.send(NetInputLagged(skipped)).await; + break; + } + Err(RecvError::Closed) => break, + } + } + }); + } + + fn stopped(&mut self, _: &mut Self::Context) { + self.signal_startup(Err( + "network event buffer stopped before startup synchronization completed".to_owned(), + )); + } +} + +impl Handler for NetEventBuffer { + type Result = (); + + fn handle(&mut self, msg: IncomingNetEvent, ctx: &mut Self::Context) { + let event_bytes = if self.state.is_running() { + 0 + } else { + msg.0.buffered_size_bytes() + }; + let result = self + .state + .observe(msg.0, event_bytes, self.max_events, self.max_bytes) + .and_then(|decision| match decision { + BufferDecision::Buffered => Ok(()), + BufferDecision::Forward(event) => self.forward_event(event), + }); + if let Err(error) = result { + self.fail_closed(error, ctx); + } + } +} + +impl Handler for NetEventBuffer { + type Result = (); + + fn handle(&mut self, msg: NetInputLagged, ctx: &mut Self::Context) { + self.fail_closed( + anyhow!( + "network event input skipped {} events because its bounded broadcast receiver lagged", + msg.0 + ), + ctx, + ); + } +} + +impl Handler for NetEventBuffer { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + if let Err(error) = self.handle_interfold_event(msg) { + self.fail_closed(error, ctx); + } + } +} diff --git a/crates/net/src/event_buffer/model.rs b/crates/net/src/event_buffer/model.rs new file mode 100644 index 0000000000..5a7aa62201 --- /dev/null +++ b/crates/net/src/event_buffer/model.rs @@ -0,0 +1,313 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::fmt::Debug; + +use std::time::Duration; + +use anyhow::{bail, ensure, Context, Result}; +use e3_events::AggregateId; +use serde::Serialize; +use tokio::time::Instant; +use tracing::info; + +use crate::{ + direct_requester::{DirectRequester, WithPeer, WithoutPeer}, + domain::sync_coordinator::effective_sync_limit, + domain::wire::{decode, MAX_DIRECT_MESSAGE_BYTES}, + events::PeerTarget, +}; + +/// Startup sync is a recovery aid, not an unbounded bulk-transfer protocol. These limits prevent +/// a peer from keeping a node in startup indefinitely or retaining an attacker-controlled history +/// in memory. Operators with a larger legitimate gap must use the controlled resync procedure. +const MAX_SYNC_FETCH_PAGES: usize = 512; +const MAX_SYNC_FETCH_EVENTS: usize = 50_000; +const MAX_SYNC_FETCH_BYTES: usize = 128 * 1024 * 1024; +const MAX_SYNC_FETCH_DURATION: Duration = Duration::from_secs(5 * 60); + +pub(crate) struct SyncFetchBudget { + started: Instant, + pages: usize, + events: usize, + bytes: usize, + max_pages: usize, + max_events: usize, + max_bytes: usize, + max_duration: Duration, + exhausted: bool, +} + +impl SyncFetchBudget { + pub(crate) fn production() -> Self { + Self::new( + MAX_SYNC_FETCH_PAGES, + MAX_SYNC_FETCH_EVENTS, + MAX_SYNC_FETCH_BYTES, + ) + } + + fn new(max_pages: usize, max_events: usize, max_bytes: usize) -> Self { + Self { + started: Instant::now(), + pages: 0, + events: 0, + bytes: 0, + max_pages, + max_events, + max_bytes, + max_duration: MAX_SYNC_FETCH_DURATION, + exhausted: false, + } + } + + pub(crate) fn is_exhausted(&self) -> bool { + self.exhausted + } + + fn remaining(&mut self) -> Result { + match self.max_duration.checked_sub(self.started.elapsed()) { + Some(remaining) => Ok(remaining), + None => { + self.exhausted = true; + bail!("historical sync exceeded total deadline") + } + } + } + + fn record_page(&mut self, event_count: usize, encoded_bytes: usize) -> Result<()> { + let pages = self + .pages + .checked_add(1) + .context("sync page count overflow")?; + let events = self + .events + .checked_add(event_count) + .context("sync event count overflow")?; + let bytes = self + .bytes + .checked_add(encoded_bytes) + .context("sync byte count overflow")?; + + if pages > self.max_pages { + self.exhausted = true; + bail!("historical sync exceeded page limit ({})", self.max_pages); + } + if events > self.max_events { + self.exhausted = true; + bail!("historical sync exceeded event limit ({})", self.max_events); + } + if bytes > self.max_bytes { + self.exhausted = true; + bail!("historical sync exceeded byte limit ({})", self.max_bytes); + } + + self.pages = pages; + self.events = events; + self.bytes = bytes; + Ok(()) + } +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub enum BatchCursor { + Done, + Next(u128), +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct EventBatch { + pub events: Vec, + pub next: BatchCursor, + pub aggregate_id: AggregateId, +} + +impl TryFrom> for EventBatch +where + E: serde::de::DeserializeOwned, +{ + type Error = anyhow::Error; + + fn try_from(value: Vec) -> Result { + decode(&value, MAX_DIRECT_MESSAGE_BYTES).context("failed to deserialize EventBatch") + } +} + +impl TryFrom> for Vec +where + E: serde::Serialize, +{ + type Error = anyhow::Error; + + fn try_from(value: EventBatch) -> Result { + bincode::serialize(&value).context("failed to serialize EventBatch") + } +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct FetchEventsSince { + aggregate_id: AggregateId, + since: u128, + limit: usize, +} + +impl FetchEventsSince { + pub fn new(aggregate_id: AggregateId, since: u128, limit: usize) -> Self { + Self { + aggregate_id, + since, + limit, + } + } + + pub fn aggregate_id(&self) -> AggregateId { + self.aggregate_id + } + + pub fn since(&self) -> u128 { + self.since + } + + pub fn limit(&self) -> usize { + self.limit + } +} + +impl TryFrom for Vec { + type Error = anyhow::Error; + + fn try_from(value: FetchEventsSince) -> Result { + bincode::serialize(&value).context("failed to serialize FetchEventsSince") + } +} + +impl TryFrom> for FetchEventsSince { + type Error = anyhow::Error; + + fn try_from(value: Vec) -> Result { + decode(&value, MAX_DIRECT_MESSAGE_BYTES).context("failed to deserialize FetchEventsSince") + } +} + +pub async fn fetch_events_since( + requester: &DirectRequester, + request: FetchEventsSince, +) -> Result> +where + E: Debug + TryFrom> + Send + Sync + 'static, + EventBatch: TryFrom>, +{ + requester.request(request).await +} + +#[cfg(test)] +async fn fetch_all_batched_events( + requester: DirectRequester, + peer: PeerTarget, + aggregate_id: AggregateId, + since: u128, + batch_size: usize, +) -> Result> +where + E: Debug + Serialize + TryFrom> + Send + Sync + 'static, + EventBatch: TryFrom>, +{ + let mut budget = SyncFetchBudget::production(); + fetch_all_batched_events_with_budget( + requester, + peer, + aggregate_id, + since, + batch_size, + &mut budget, + ) + .await +} + +pub(crate) async fn fetch_all_batched_events_with_budget( + requester: DirectRequester, + peer: PeerTarget, + aggregate_id: AggregateId, + since: u128, + batch_size: usize, + budget: &mut SyncFetchBudget, +) -> Result> +where + E: Debug + Serialize + TryFrom> + Send + Sync + 'static, + EventBatch: TryFrom>, +{ + ensure!(batch_size > 0, "sync batch size must be greater than 0"); + let batch_size = effective_sync_limit(batch_size); + let requester = requester.to(peer); + let mut all_events = Vec::new(); + let mut cursor = since; + + loop { + let request = FetchEventsSince::new(aggregate_id, cursor, batch_size); + info!( + "Fetching batch aggregate={} cursor={} batch_size={}", + aggregate_id, cursor, batch_size + ); + let remaining = budget.remaining()?; + let batch = + match tokio::time::timeout(remaining, fetch_events_since(&requester, request)).await { + Ok(result) => result?, + Err(_) => { + budget.exhausted = true; + bail!("historical sync exceeded total deadline"); + } + }; + ensure!( + batch.aggregate_id == aggregate_id, + "sync peer returned aggregate {} while fetching {}", + batch.aggregate_id, + aggregate_id + ); + ensure!( + batch.events.len() <= batch_size, + "sync peer returned {} events, exceeding requested batch size {}", + batch.events.len(), + batch_size + ); + info!( + "Batch received with {} events for aggregate={} cursor={}", + batch.events.len(), + aggregate_id, + cursor + ); + + let page_bytes: usize = bincode::serialized_size(&batch.events) + .context("failed to measure historical sync page")? + .try_into() + .context("historical sync page size does not fit usize")?; + budget.record_page(batch.events.len(), page_bytes)?; + all_events + .try_reserve(batch.events.len()) + .map_err(|error| { + anyhow::anyhow!("failed to reserve historical sync buffer: {error}") + })?; + all_events.extend(batch.events); + + match batch.next { + BatchCursor::Done => break, + BatchCursor::Next(next_cursor) => { + ensure!( + next_cursor > cursor, + "sync peer returned non-advancing cursor {next_cursor} from {cursor}" + ); + cursor = next_cursor; + } + } + } + + info!("Batch is done returning {} events", all_events.len()); + + Ok(all_events) +} + +#[cfg(test)] +#[path = "model_tests.rs"] +mod tests; diff --git a/crates/net/src/event_buffer/model_tests.rs b/crates/net/src/event_buffer/model_tests.rs new file mode 100644 index 0000000000..b7bbe79502 --- /dev/null +++ b/crates/net/src/event_buffer/model_tests.rs @@ -0,0 +1,276 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use crate::direct_requester::DirectRequesterTester; +use crate::events::{NetCommand, NetEvent, PeerTarget}; +use std::sync::Arc; +use tokio::sync::{broadcast, mpsc}; + +#[test] +fn sync_fetch_budget_rejects_each_resource_limit_without_mutating_state() { + let mut budget = SyncFetchBudget::new(1, 2, 3); + budget.record_page(2, 3).unwrap(); + + let page_error = budget.record_page(0, 0).unwrap_err(); + assert!(page_error.to_string().contains("page limit")); + assert_eq!((budget.pages, budget.events, budget.bytes), (1, 2, 3)); + + let mut event_budget = SyncFetchBudget::new(2, 1, 3); + let event_error = event_budget.record_page(2, 0).unwrap_err(); + assert!(event_error.to_string().contains("event limit")); + assert_eq!( + (event_budget.pages, event_budget.events, event_budget.bytes), + (0, 0, 0) + ); + + let mut byte_budget = SyncFetchBudget::new(2, 2, 1); + let byte_error = byte_budget.record_page(0, 2).unwrap_err(); + assert!(byte_error.to_string().contains("byte limit")); + assert_eq!( + (byte_budget.pages, byte_budget.events, byte_budget.bytes), + (0, 0, 0) + ); +} + +#[tokio::test] +async fn test_fetch_all_batched_events() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + + let batch1 = EventBatch { + events: vec![b"event1".to_vec(), b"event2".to_vec()], + next: BatchCursor::Next(100), + aggregate_id: AggregateId::new(1), + }; + let batch2 = EventBatch { + events: vec![b"event3".to_vec()], + next: BatchCursor::Done, + aggregate_id: AggregateId::new(1), + }; + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 100)) + .respond_with(batch1) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 100, 100)) + .respond_with(batch2) + .spawn(); + + let events: Vec> = + fetch_all_batched_events(requester, PeerTarget::Random, AggregateId::new(1), 0, 100) + .await + .unwrap(); + + handle.await.unwrap(); + + assert_eq!( + events, + vec![b"event1".to_vec(), b"event2".to_vec(), b"event3".to_vec(),] + ); +} + +#[tokio::test] +async fn test_no_duplicate_events_across_batches() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + + let batch1 = EventBatch { + events: vec![b"event1".to_vec(), b"event2".to_vec()], + next: BatchCursor::Next(2), + aggregate_id: AggregateId::new(1), + }; + let batch2 = EventBatch { + events: vec![b"event3".to_vec()], + next: BatchCursor::Done, + aggregate_id: AggregateId::new(1), + }; + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 100)) + .respond_with(batch1) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 2, 100)) + .respond_with(batch2) + .spawn(); + + let events: Vec> = + fetch_all_batched_events(requester, PeerTarget::Random, AggregateId::new(1), 0, 100) + .await + .unwrap(); + + handle.await.unwrap(); + + let unique: Vec<_> = events + .iter() + .collect::>() + .into_iter() + .collect(); + assert_eq!( + events.len(), + unique.len(), + "Found duplicate events: total={}, unique={}", + events.len(), + unique.len() + ); + assert_eq!( + events, + vec![b"event1".to_vec(), b"event2".to_vec(), b"event3".to_vec()] + ); +} + +#[tokio::test] +async fn test_cursor_correctly_excludes_last_event_on_batch_boundary() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + + let batch1 = EventBatch { + events: vec![b"event1".to_vec()], + next: BatchCursor::Next(100), + aggregate_id: AggregateId::new(1), + }; + let batch2 = EventBatch { + events: vec![b"event2".to_vec()], + next: BatchCursor::Done, + aggregate_id: AggregateId::new(1), + }; + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 1)) + .respond_with(batch1) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 100, 1)) + .respond_with(batch2) + .spawn(); + + let events: Vec> = + fetch_all_batched_events(requester, PeerTarget::Random, AggregateId::new(1), 0, 1) + .await + .unwrap(); + + handle.await.unwrap(); + + assert_eq!(events.len(), 2); + assert_eq!(events[0], b"event1"); + assert_eq!(events[1], b"event2"); +} + +#[tokio::test] +async fn test_non_advancing_cursor_is_rejected() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + let batch = EventBatch { + events: vec![b"event1".to_vec()], + next: BatchCursor::Next(0), + aggregate_id: AggregateId::new(1), + }; + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 1)) + .respond_with(batch) + .spawn(); + + let error = fetch_all_batched_events::>( + requester, + PeerTarget::Random, + AggregateId::new(1), + 0, + 1, + ) + .await + .unwrap_err(); + + handle.await.unwrap(); + assert!(error.to_string().contains("non-advancing cursor")); +} + +#[tokio::test] +async fn test_single_batch_with_limit_exactly_matched_returns_done() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + + let batch = EventBatch { + events: vec![b"event1".to_vec()], + next: BatchCursor::Done, + aggregate_id: AggregateId::new(1), + }; + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 1)) + .respond_with(batch) + .spawn(); + + let events: Vec> = + fetch_all_batched_events(requester, PeerTarget::Random, AggregateId::new(1), 0, 1) + .await + .unwrap(); + + handle.await.unwrap(); + + assert_eq!(events, vec![b"event1".to_vec()]); +} + +#[tokio::test] +async fn test_three_batches_with_cursor_continuity() { + let (net_cmds_tx, net_cmds_rx) = mpsc::channel::(16); + let (net_events_tx, net_events_rx) = broadcast::channel::(16); + let net_events = Arc::new(net_events_rx); + + let requester = DirectRequester::builder(net_cmds_tx, net_events).build(); + + let batch1 = EventBatch { + events: vec![b"a".to_vec(), b"b".to_vec()], + next: BatchCursor::Next(200), + aggregate_id: AggregateId::new(1), + }; + let batch2 = EventBatch { + events: vec![b"c".to_vec(), b"d".to_vec()], + next: BatchCursor::Next(400), + aggregate_id: AggregateId::new(1), + }; + let batch3 = EventBatch { + events: vec![b"e".to_vec()], + next: BatchCursor::Done, + aggregate_id: AggregateId::new(1), + }; + + let handle = DirectRequesterTester::new(net_cmds_rx, net_events_tx) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 0, 2)) + .respond_with(batch1) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 200, 2)) + .respond_with(batch2) + .expect_request(FetchEventsSince::new(AggregateId::new(1), 400, 2)) + .respond_with(batch3) + .spawn(); + + let events: Vec> = + fetch_all_batched_events(requester, PeerTarget::Random, AggregateId::new(1), 0, 2) + .await + .unwrap(); + + handle.await.unwrap(); + + let expected = vec![ + b"a".to_vec(), + b"b".to_vec(), + b"c".to_vec(), + b"d".to_vec(), + b"e".to_vec(), + ]; + assert_eq!(events.len(), expected.len()); + assert_eq!(events, expected); +} diff --git a/crates/net/src/event_buffer/tests.rs b/crates/net/src/event_buffer/tests.rs new file mode 100644 index 0000000000..b08c190a29 --- /dev/null +++ b/crates/net/src/event_buffer/tests.rs @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::time::Duration; + +use super::*; +use crate::events::{GossipData, NetEvent}; +use e3_ciphernode_builder::EventSystem; +use e3_events::EventPublisher; +use e3_events::SyncEnded; +use tokio::{ + sync::broadcast, + time::{sleep, timeout}, +}; + +#[actix::test] +async fn test_buffers_until_sync_ended() -> Result<()> { + // Setup + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test"); + let (input_tx, input_rx) = broadcast::channel(16); + let (mut output_rx, handle) = NetEventBuffer::setup_with_limits( + &bus, + &input_rx, + DEFAULT_MAX_BUFFERED_NET_EVENTS, + DEFAULT_MAX_BUFFERED_NET_BYTES, + ); + + // Send events while syncing - should be buffered + let event1 = NetEvent::GossipData(GossipData::GossipBytes(vec![1, 2, 3])); + let event2 = NetEvent::GossipData(GossipData::GossipBytes(vec![4, 5, 6])); + input_tx.send(event1.clone()).unwrap(); + input_tx.send(event2.clone()).unwrap(); + + // Give actor time to process + sleep(Duration::from_millis(10)).await; + + // Verify no events forwarded yet (should timeout) + assert!( + timeout(Duration::from_millis(50), output_rx.recv()) + .await + .is_err(), + "Events should be buffered, not forwarded during sync" + ); + + // Send SyncEnded event + bus.publish_without_context(SyncEnded::new()).unwrap(); + handle.wait_until_running().await?; + + // Now buffered events should be forwarded + let received1 = output_rx.recv().await.unwrap(); + let received2 = output_rx.recv().await.unwrap(); + + assert!( + matches!(received1, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![1, 2, 3]) + ); + assert!( + matches!(received2, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![4, 5, 6]) + ); + + // Send new event after sync - should forward immediately + let event3 = NetEvent::GossipData(GossipData::GossipBytes(vec![7, 8, 9])); + input_tx.send(event3.clone()).unwrap(); + + let received3 = tokio::time::timeout(tokio::time::Duration::from_millis(100), output_rx.recv()) + .await + .expect("Event should be forwarded immediately after sync") + .unwrap(); + + assert!( + matches!(received3, NetEvent::GossipData(GossipData::GossipBytes(ref bytes)) if bytes == &vec![7, 8, 9]) + ); + + Ok(()) +} + +#[actix::test] +async fn startup_buffer_overflow_fails_readiness_without_dropping_oldest() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-overflow"); + let (input_tx, input_rx) = broadcast::channel(16); + let (_output_rx, handle) = + NetEventBuffer::setup_with_limits(&bus, &input_rx, 1, DEFAULT_MAX_BUFFERED_NET_BYTES); + + input_tx.send(NetEvent::GossipData(GossipData::GossipBytes(vec![1])))?; + input_tx.send(NetEvent::GossipData(GossipData::GossipBytes(vec![2])))?; + + let error = timeout(Duration::from_secs(1), handle.wait_until_running()) + .await + .context("network buffer did not report overflow")? + .expect_err("overflow must fail startup readiness") + .to_string(); + assert!(error.contains("events=1/1"), "{error}"); + assert!( + error.contains("startup will stop rather than drop"), + "{error}" + ); + Ok(()) +} + +#[actix::test] +async fn startup_buffer_enforces_estimated_payload_bytes() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-byte-overflow"); + let (input_tx, input_rx) = broadcast::channel(16); + let event = NetEvent::GossipData(GossipData::GossipBytes(vec![0; 32])); + let estimated_bytes = event.buffered_size_bytes(); + let (_output_rx, handle) = + NetEventBuffer::setup_with_limits(&bus, &input_rx, 16, estimated_bytes - 1); + + input_tx.send(event)?; + + let error = timeout(Duration::from_secs(1), handle.wait_until_running()) + .await + .context("network buffer did not report byte overflow")? + .expect_err("byte overflow must fail startup readiness") + .to_string(); + assert!( + error.contains(&format!("next_event_bytes={estimated_bytes}")), + "{error}" + ); + Ok(()) +} diff --git a/crates/net/src/domain/net_buffer.rs b/crates/net/src/event_buffer/workflow.rs similarity index 100% rename from crates/net/src/domain/net_buffer.rs rename to crates/net/src/event_buffer/workflow.rs diff --git a/crates/net/src/event_conversion/actor.rs b/crates/net/src/event_conversion/actor.rs new file mode 100644 index 0000000000..05bdc08ac0 --- /dev/null +++ b/crates/net/src/event_conversion/actor.rs @@ -0,0 +1,168 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Translate between typed protocol artifacts and network documents. + +use crate::domain::{EventConversionService, IncomingDocument}; +use actix::prelude::*; +use anyhow::Result; +use e3_events::{ + prelude::*, trap, BusHandle, DecryptionKeyShared, DocumentReceived, EType, + EncryptionKeyCreated, EventType, InterfoldEvent, InterfoldEventData, ThresholdShareCreated, + TypedEvent, +}; +use e3_utils::NotifySync; + +/// Converts between internal events and network documents. +/// +/// Responsibilities: +/// - Outgoing: Converts ThresholdShareCreated → party-filtered PublishDocumentRequested +/// - Incoming: Converts DocumentReceived → ThresholdShareCreated/EncryptionKeyCreated +/// +/// The conversion logic lives in [`EventConversionService`]; this actor only publishes the +/// resulting events on the bus. +pub struct EventConverter { + bus: BusHandle, +} + +impl EventConverter { + pub fn new(bus: &BusHandle) -> Self { + Self { bus: bus.clone() } + } + + pub fn setup(bus: &BusHandle) -> Addr { + let addr = Self::new(bus).start(); + bus.subscribe(EventType::ThresholdShareCreated, addr.clone().into()); + bus.subscribe(EventType::EncryptionKeyCreated, addr.clone().into()); + bus.subscribe(EventType::DecryptionKeyShared, addr.clone().into()); + bus.subscribe(EventType::DocumentReceived, addr.clone().into()); + addr + } + + fn handle_threshold_share_created(&self, msg: TypedEvent) -> Result<()> { + let (msg, ctx) = msg.into_components(); + if let Some(request) = EventConversionService::threshold_share_to_request(msg)? { + self.bus.publish(request, ctx)?; + } + Ok(()) + } + + fn handle_encryption_key_created(&self, msg: TypedEvent) -> Result<()> { + let (msg, ctx) = msg.into_components(); + if let Some(request) = EventConversionService::encryption_key_to_request(msg)? { + self.bus.publish(request, ctx)?; + } + Ok(()) + } + + fn handle_decryption_key_shared(&self, msg: TypedEvent) -> Result<()> { + let (msg, ctx) = msg.into_components(); + if let Some(request) = EventConversionService::decryption_key_to_request(msg)? { + self.bus.publish(request, ctx)?; + } + Ok(()) + } + + /// Convert received document to internal events. + /// Note: Filtering already happened in DocumentPublisher before DHT fetch. + fn handle_document_received(&self, msg: TypedEvent) -> Result<()> { + let (msg, ctx) = msg.into_components(); + match EventConversionService::decode_received(&msg.meta, &msg.value)? { + IncomingDocument::ThresholdShare(evt) => { + self.bus.publish(evt, ctx)?; + } + IncomingDocument::EncryptionKey(evt) => { + self.bus.publish(evt, ctx)?; + } + IncomingDocument::DecryptionKey(evt) => { + self.bus.publish(evt, ctx)?; + } + } + Ok(()) + } +} + +impl Actor for EventConverter { + type Context = actix::Context; +} + +impl Handler for EventConverter { + type Result = (); + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (data, ec) = msg.into_components(); + match data { + InterfoldEventData::ThresholdShareCreated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::EncryptionKeyCreated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::DecryptionKeyShared(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::DocumentReceived(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + _ => (), + } + } +} + +impl Handler> for EventConverter { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::DocumentPublishing, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_threshold_share_created(msg), + ) + } +} + +impl Handler> for EventConverter { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::DocumentPublishing, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_encryption_key_created(msg), + ) + } +} + +impl Handler> for EventConverter { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::DocumentPublishing, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_decryption_key_shared(msg), + ) + } +} + +impl Handler> for EventConverter { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap( + EType::DocumentPublishing, + &self.bus.with_ec(msg.get_ctx()), + || self.handle_document_received(msg), + ) + } +} diff --git a/crates/net/src/domain/event_conversion.rs b/crates/net/src/event_conversion/workflow.rs similarity index 100% rename from crates/net/src/domain/event_conversion.rs rename to crates/net/src/event_conversion/workflow.rs diff --git a/crates/net/src/actors/mod.rs b/crates/net/src/event_subscription.rs similarity index 80% rename from crates/net/src/actors/mod.rs rename to crates/net/src/event_subscription.rs index 16cfe260bc..ac32b5e0c4 100644 --- a/crates/net/src/actors/mod.rs +++ b/crates/net/src/event_subscription.rs @@ -4,31 +4,12 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -//! Thin actix actor shells. These contain no business logic — they wire bus/network events to the -//! pure services in [`crate::domain`] and perform the resulting I/O. +//! Shared retained-event subscription behavior for network capabilities. use crate::events::NetEvent; use tokio::sync::broadcast::{error::RecvError, Receiver}; use tracing::{debug, warn}; -mod document_publisher; -mod net_event_buffer; -mod net_event_translator; -mod net_sync_manager; - -pub use document_publisher::{ - handle_document_published_notification, handle_publish_document_requested, DocumentPublisher, - EventConverter, -}; -pub use net_event_buffer::{ - NetEventBufferHandle, DEFAULT_MAX_BUFFERED_NET_BYTES, DEFAULT_MAX_BUFFERED_NET_EVENTS, -}; -pub use net_event_translator::NetEventTranslator; - -// Internal wiring helpers used by `setup_net`; not part of the public API. -pub(crate) use net_event_buffer::NetEventBuffer; -pub(crate) use net_sync_manager::NetSyncManager; - /// Receive the next retained network event without treating broadcast lag as channel closure. /// /// Tokio reports lag as an item-level error and advances the receiver to the oldest event still in @@ -36,7 +17,7 @@ pub(crate) use net_sync_manager::NetSyncManager; /// would permanently detach that networking subsystem after a single burst. Only `Closed` is /// terminal. The warning deliberately contains only a static consumer name and a numeric count so /// peer- or payload-controlled data cannot create unbounded log fields. -async fn recv_net_event( +pub(crate) async fn recv_net_event( receiver: &mut Receiver, consumer: &'static str, ) -> Option { diff --git a/crates/net/src/actors/net_event_translator.rs b/crates/net/src/event_translation/actor.rs similarity index 96% rename from crates/net/src/actors/net_event_translator.rs rename to crates/net/src/event_translation/actor.rs index 5b162f994d..681e19c357 100644 --- a/crates/net/src/actors/net_event_translator.rs +++ b/crates/net/src/event_translation/actor.rs @@ -63,7 +63,9 @@ impl NetEventTranslator { tokio::spawn({ let addr = addr.clone(); async move { - while let Some(event) = super::recv_net_event(&mut rx, "NetEventTranslator").await { + while let Some(event) = + crate::event_subscription::recv_net_event(&mut rx, "NetEventTranslator").await + { if let NetEvent::GossipData(data) = event { if let GossipData::GossipBytes(_) = data { if let Err(error) = addr.send(LibP2pEvent(data)).await { diff --git a/crates/net/src/domain/event_translation.rs b/crates/net/src/event_translation/workflow.rs similarity index 100% rename from crates/net/src/domain/event_translation.rs rename to crates/net/src/event_translation/workflow.rs diff --git a/crates/net/src/lib.rs b/crates/net/src/lib.rs index 6b305e3b88..94fccb0370 100644 --- a/crates/net/src/lib.rs +++ b/crates/net/src/lib.rs @@ -10,6 +10,7 @@ mod dialer; pub mod direct_requester; pub mod direct_responder; mod domain; +mod event_subscription; pub mod events; mod keypair; mod net_interface; diff --git a/crates/net/src/domain/network_status.rs b/crates/net/src/network_status.rs similarity index 100% rename from crates/net/src/domain/network_status.rs rename to crates/net/src/network_status.rs diff --git a/crates/net/src/network_sync/actor.rs b/crates/net/src/network_sync/actor.rs new file mode 100644 index 0000000000..6e0302c2ad --- /dev/null +++ b/crates/net/src/network_sync/actor.rs @@ -0,0 +1,163 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use actix::{Actor, Addr, AsyncContext, Handler, Message, Recipient, ResponseFuture}; +use anyhow::{bail, Context, Result}; +use e3_events::{ + prelude::*, trap, trap_fut, AggregateId, BusHandle, CorrelationId, EType, EventSource, + EventStoreFilter, EventStoreQueryBy, EventStoreQueryResponse, EventType, + HistoricalNetSyncEventsReceived, HistoricalNetSyncStart, InterfoldEvent, InterfoldEventData, + NetReady, TsAgg, TypedEvent, Unsequenced, +}; +use e3_utils::MAILBOX_LIMIT; +use libp2p::PeerId; +use serde::{Deserialize, Serialize}; +use std::{ + collections::{HashMap, HashSet}, + convert::TryInto, + sync::Arc, + time::Duration, +}; +use tokio::sync::{broadcast, mpsc}; +use tracing::{debug, error, info, warn}; + +use crate::{ + direct_requester::DirectRequester, + direct_responder::DirectResponder, + domain::{ + build_sync_batch, + net_event_batch::{ + fetch_all_batched_events_with_budget, FetchEventsSince, SyncFetchBudget, + }, + sync_coordinator::sync_scan_limit, + wire::{decode, MAX_DIRECT_MESSAGE_BYTES}, + EventTranslationService, NetReadiness, ReadinessDecision, SyncBatchOutcome, + }, + events::{ + await_event, GossipData, IncomingRequest, NetCommand, NetEvent, PeerTarget, + ProtocolResponse, + }, +}; + +/// Maximum time to wait for a `ConnectionEstablished` event after all dials +/// failed before publishing `NetReady` anyway. +const NET_READY_CONNECT_TIMEOUT: Duration = Duration::from_secs(60); + +/// Direct-request retry settings for a single historical sync fetch attempt. +const SYNC_FETCH_MAX_RETRIES: u32 = 3; +const SYNC_FETCH_RETRY_TIMEOUT: Duration = Duration::from_secs(5); + +/// If a historical sync fetch fails, wait this long for a fresh connection +/// before retrying anyway against currently connected peers. +const SYNC_RECOVERY_RETRY_INTERVAL: Duration = Duration::from_secs(15); + +/// Number of recovery rounds to try for failed aggregates after the initial fetch pass. +const SYNC_RECOVERY_MAX_ATTEMPTS: usize = 3; + +/// Bound remote work independently of the actor mailbox. Per-peer admission prevents one +/// authenticated transport identity from occupying the global allowance. +const MAX_IN_FLIGHT_SYNC_REQUESTS: usize = 16; +const MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER: usize = 2; + +/// Expire storage requests before libp2p's 30-second request timeout so a failed local query cannot +/// retain a responder and permanently consume one of the bounded in-flight slots. +const INCOMING_SYNC_REQUEST_TIMEOUT: Duration = Duration::from_secs(25); + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SyncResponseValue { + pub events: Vec>, + pub ts: u128, +} + +impl TryInto> for SyncResponseValue { + type Error = anyhow::Error; + + fn try_into(self) -> Result, Self::Error> { + bincode::serialize(&self).context("failed to serialize sync response") + } +} + +impl TryFrom> for SyncResponseValue { + type Error = anyhow::Error; + + fn try_from(value: Vec) -> Result { + decode(&value, MAX_DIRECT_MESSAGE_BYTES).context("failed to deserialize sync response") + } +} + +#[derive(Debug, Clone)] +pub struct SyncRequestSucceeded { + pub response: SyncResponseValue, +} + +struct PendingSyncRequest { + peer: PeerId, + responder: DirectResponder, +} + +pub struct NetSyncManager { + /// Interfold EventBus + bus: BusHandle, + /// NetCommand sender to forward commands to the Libp2pNetInterface + tx: mpsc::Sender, + /// NetEvents receiver to receive events + rx: Arc>, + eventstore: Recipient>, + requests: HashMap, + /// Pure readiness state machine. + readiness: NetReadiness, + /// Gossipsub topic used to re-broadcast our own forwardable artifacts after a restart. + topic: String, + /// Snapshot-cursor map captured from `HistoricalNetSyncStart`. Bounds the post-restart + /// re-broadcast query to the in-flight (un-snapshotted) window. + rebroadcast_since: Option>, + /// Correlation ids of in-flight re-broadcast EventStore queries, so their responses can be + /// distinguished from ordinary sync-request responses. + rebroadcast_query_ids: HashSet, + /// Set once `NetReady` has been published (peers connected or fallback timeout elapsed). + net_ready: bool, + /// Guard so the post-restart re-broadcast fires at most once per process. + rebroadcast_started: bool, +} + +impl NetSyncManager { + pub fn new( + bus: &BusHandle, + tx: &mpsc::Sender, + rx: &Arc>, + eventstore: Recipient>, + topic: &str, + ) -> Self { + Self { + bus: bus.clone(), + tx: tx.clone(), + rx: Arc::clone(rx), + eventstore, + requests: HashMap::new(), + readiness: NetReadiness::new(), + topic: topic.to_string(), + rebroadcast_since: None, + rebroadcast_query_ids: HashSet::new(), + net_ready: false, + rebroadcast_started: false, + } + } +} + +#[path = "effects/mod.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +use effects::historical_sync::handle_sync_request_event; +use handlers::{AllPeersDialed, PeerConnected}; + +#[cfg(test)] +use effects::historical_sync::validate_historical_events; + +#[cfg(test)] +#[path = "tests.rs"] +mod tests; diff --git a/crates/net/src/domain/correlator.rs b/crates/net/src/network_sync/correlator.rs similarity index 100% rename from crates/net/src/domain/correlator.rs rename to crates/net/src/network_sync/correlator.rs diff --git a/crates/net/src/network_sync/effects/admission.rs b/crates/net/src/network_sync/effects/admission.rs new file mode 100644 index 0000000000..fedb593f41 --- /dev/null +++ b/crates/net/src/network_sync/effects/admission.rs @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Bound incoming historical-sync work and expire abandoned storage requests. + +use super::*; + +impl NetSyncManager { + pub(in crate::actors::net_sync_manager) fn publish_net_ready(&self) -> Result<()> { + info!("NetSyncManager: publishing NetReady"); + self.bus.publish_without_context(NetReady::new())?; + Ok(()) + } + + pub(in crate::actors::net_sync_manager) fn request_capacity_error( + &self, + peer: &PeerId, + ) -> Option<&'static str> { + if self.requests.len() >= MAX_IN_FLIGHT_SYNC_REQUESTS { + return Some("too many in-flight sync requests"); + } + let peer_requests = self + .requests + .values() + .filter(|request| &request.peer == peer) + .count(); + if peer_requests >= MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER { + return Some("too many in-flight sync requests from this peer"); + } + None + } + + pub(in crate::actors::net_sync_manager) fn expire_sync_request(&mut self, id: CorrelationId) { + let Some(pending) = self.requests.remove(&id) else { + return; + }; + warn!( + peer = %pending.peer, + correlation_id = %id, + timeout_ms = INCOMING_SYNC_REQUEST_TIMEOUT.as_millis(), + "Incoming historical-sync storage query timed out" + ); + if let Err(error) = pending.responder.respond(ProtocolResponse::Error( + "historical sync request timed out".to_string(), + )) { + warn!( + peer = %pending.peer, + correlation_id = %id, + %error, + "Failed to send historical-sync timeout response" + ); + } + } +} diff --git a/crates/net/src/network_sync/effects/fetch_history.rs b/crates/net/src/network_sync/effects/fetch_history.rs new file mode 100644 index 0000000000..06514fcfe9 --- /dev/null +++ b/crates/net/src/network_sync/effects/fetch_history.rs @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Bounded historical-event fetching, validation, and recovery retries. + +use super::*; + +pub(in crate::actors::net_sync_manager) async fn fetch_historical_events_for_aggregate( + net_cmds: &mpsc::Sender, + net_events: &Arc>, + aggregate_id: AggregateId, + since: u128, + budget: &mut SyncFetchBudget, +) -> Result>> { + let requester = DirectRequester::builder(net_cmds.clone(), net_events.clone()) + .max_retries(SYNC_FETCH_MAX_RETRIES) + .retry_timeout(SYNC_FETCH_RETRY_TIMEOUT) + .build(); + + let events = fetch_all_batched_events_with_budget::>( + requester, + PeerTarget::Random, + aggregate_id, + since, + 100, + budget, + ) + .await?; + + validate_historical_events(aggregate_id, events) +} + +pub(in crate::actors::net_sync_manager) fn validate_historical_events( + aggregate_id: AggregateId, + events: Vec>, +) -> Result>> { + for event in &events { + if event.aggregate_id() != aggregate_id { + bail!( + "historical sync peer returned event for aggregate {} while fetching {}", + event.aggregate_id(), + aggregate_id + ); + } + if !EventTranslationService::is_forwardable_event(event) { + bail!( + "historical sync peer returned non-forwardable event type {}", + event.event_type() + ); + } + } + Ok(events) +} + +pub(in crate::actors::net_sync_manager) async fn handle_sync_request_event( + net_cmds: mpsc::Sender, + net_events: Arc>, + event: TypedEvent, + address: impl Into>>, + wait_for_event: bool, +) -> Result<()> { + info!("Sync request event received"); + let (event, ctx) = event.into_components(); + info!("Checking for AllPeersDialed..."); + if wait_for_event { + info!("Waiting for peer connection..."); + let has_peers = await_event( + &net_events, + |e| match e { + NetEvent::ConnectionEstablished { .. } => { + info!("Peer connection established"); + Some(true) + } + NetEvent::AllPeersDialed { total: 0, .. } => { + info!("No peers configured, proceeding without sync"); + Some(false) + } + _ => None, + }, + NET_READY_CONNECT_TIMEOUT, + ) + .await + .context("No peer connections established within timeout")?; + + if !has_peers { + let value = SyncRequestSucceeded { + response: SyncResponseValue { + events: vec![], + ts: 0, + }, + }; + + address.into().try_send(TypedEvent::new(value, ctx))?; + return Ok(()); + } + } + info!("handle_sync_request_event: ready to sync"); + + let mut all_events: Vec> = Vec::new(); + let mut latest_timestamp: u128 = 0; + let mut failed_aggregates: Vec = Vec::new(); + let mut budget = SyncFetchBudget::production(); + + for (aggregate_id, since) in event.since.iter() { + info!( + "Requesting batched events for aggregate_id={} since={}", + aggregate_id, since + ); + match fetch_historical_events_for_aggregate( + &net_cmds, + &net_events, + *aggregate_id, + *since, + &mut budget, + ) + .await + { + Ok(events) => { + info!( + "Received {} events for aggregate_id={}", + events.len(), + aggregate_id + ); + for interfold_event in events { + let ts = interfold_event.ts(); + if ts > latest_timestamp { + latest_timestamp = ts; + } + all_events.push(interfold_event); + } + } + Err(e) => { + if budget.is_exhausted() { + return Err(e).context("historical net sync exhausted its global budget"); + } + warn!( + "Failed to fetch events for aggregate_id={}: {e}. Continuing with available events.", + aggregate_id + ); + failed_aggregates.push(*aggregate_id); + } + } + } + + // If any aggregate failed, retry a few recovery rounds. Prefer a fresh + // ConnectionEstablished signal when one arrives, but do not depend on it: + // a connected peer may simply be slow or temporarily stalled. + if !failed_aggregates.is_empty() { + info!( + "Sync fetch failed for {} aggregates — starting recovery retries...", + failed_aggregates.len() + ); + let mut recovery_attempt = 0; + + while !failed_aggregates.is_empty() && recovery_attempt < SYNC_RECOVERY_MAX_ATTEMPTS { + recovery_attempt += 1; + + match await_event( + &net_events, + |e| { + if matches!(e, NetEvent::ConnectionEstablished { .. }) { + Some(()) + } else { + None + } + }, + SYNC_RECOVERY_RETRY_INTERVAL, + ) + .await + { + Ok(()) => { + info!( + attempt = recovery_attempt, + "Peer reconnected, retrying failed aggregates" + ); + } + Err(_) => { + info!( + attempt = recovery_attempt, + retry_after = ?SYNC_RECOVERY_RETRY_INTERVAL, + "No new peer connection observed; retrying failed aggregates against current peers" + ); + } + } + + let mut still_failed = Vec::new(); + for aggregate_id in failed_aggregates { + let since = event.since.get(&aggregate_id).copied().unwrap_or(0); + match fetch_historical_events_for_aggregate( + &net_cmds, + &net_events, + aggregate_id, + since, + &mut budget, + ) + .await + { + Ok(events) => { + info!( + attempt = recovery_attempt, + "Retry succeeded: {} events for aggregate_id={}", + events.len(), + aggregate_id + ); + for interfold_event in events { + let ts = interfold_event.ts(); + if ts > latest_timestamp { + latest_timestamp = ts; + } + all_events.push(interfold_event); + } + } + Err(e) => { + if budget.is_exhausted() { + return Err(e) + .context("historical net sync exhausted its global budget"); + } + warn!( + attempt = recovery_attempt, + "Retry failed for aggregate_id={}: {e}", aggregate_id + ); + still_failed.push(aggregate_id); + } + } + } + + failed_aggregates = still_failed; + } + + if !failed_aggregates.is_empty() { + bail!( + "failed to fetch historical net events for aggregates: {:?} after {} recovery attempts", + failed_aggregates, + SYNC_RECOVERY_MAX_ATTEMPTS + ); + } + } + + info!( + "Sync complete: collected {} events across {} aggregates, latest_timestamp={}", + all_events.len(), + event.since.len(), + latest_timestamp + ); + + let value = SyncRequestSucceeded { + response: SyncResponseValue { + events: all_events, + ts: latest_timestamp, + }, + }; + + address.into().try_send(TypedEvent::new(value, ctx))?; + Ok(()) +} diff --git a/crates/net/src/network_sync/effects/mod.rs b/crates/net/src/network_sync/effects/mod.rs new file mode 100644 index 0000000000..16f3b57365 --- /dev/null +++ b/crates/net/src/network_sync/effects/mod.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Bounded network effects and lifecycle coordination. + +use super::*; + +mod admission; +#[path = "fetch_history.rs"] +pub(super) mod historical_sync; +mod readiness; +mod rebroadcast; diff --git a/crates/net/src/network_sync/effects/readiness.rs b/crates/net/src/network_sync/effects/readiness.rs new file mode 100644 index 0000000000..3c08cd8d80 --- /dev/null +++ b/crates/net/src/network_sync/effects/readiness.rs @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Own network readiness, fallback timing, and ingress task setup. + +use super::*; + +impl NetSyncManager { + /// Apply a readiness decision: publish `NetReady`, or schedule the fallback timeout. + pub(in crate::actors::net_sync_manager) fn apply_readiness( + &mut self, + decision: ReadinessDecision, + ctx: &mut actix::Context, + ) { + match decision { + ReadinessDecision::PublishReady => { + if let Err(e) = self.publish_net_ready() { + error!("Failed to publish NetReady: {e}"); + } + self.net_ready = true; + self.maybe_rebroadcast_own_artifacts(ctx); + } + ReadinessDecision::WaitForConnection => { + info!( + "All peer dials failed, waiting for connections before publishing NetReady..." + ); + ctx.run_later(NET_READY_CONNECT_TIMEOUT, move |this, ctx| { + if let ReadinessDecision::PublishReady = this.readiness.on_connect_timeout() { + warn!("No peer connections established within 60s timeout, publishing NetReady anyway"); + if let Err(e) = this.publish_net_ready() { + error!("Failed to publish NetReady: {e}"); + } + this.net_ready = true; + this.maybe_rebroadcast_own_artifacts(ctx); + } + }); + } + ReadinessDecision::Idle => {} + } + } + + pub fn setup( + bus: &BusHandle, + tx: &mpsc::Sender, + rx: &Arc>, + eventstore: Recipient>, + topic: &str, + ) -> Addr { + let mut events = rx.resubscribe(); + let addr = Self::new(bus, tx, rx, eventstore, topic).start(); + + bus.subscribe(EventType::HistoricalNetSyncStart, addr.clone().recipient()); + + // Forward from NetEvent + tokio::spawn({ + debug!("Spawning event receive loop!"); + let addr = addr.clone(); + async move { + while let Some(event) = + crate::event_subscription::recv_net_event(&mut events, "NetSyncManager").await + { + debug!("Received event {:?}", event); + let delivery = match event { + // Someone is asking for our sync + NetEvent::IncomingRequest(value) => addr.send(value).await, + NetEvent::AllPeersDialed { connected, total } => { + addr.send(AllPeersDialed { connected, total }).await + } + NetEvent::ConnectionEstablished { .. } => addr.send(PeerConnected).await, + _ => continue, + }; + if let Err(error) = delivery { + warn!(%error, "NetSyncManager stopped; ending sync ingress"); + break; + } + } + } + }); + + addr + } +} diff --git a/crates/net/src/network_sync/effects/rebroadcast.rs b/crates/net/src/network_sync/effects/rebroadcast.rs new file mode 100644 index 0000000000..766105ad65 --- /dev/null +++ b/crates/net/src/network_sync/effects/rebroadcast.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Re-gossip this node's forwardable in-flight artifacts after restart. + +use super::*; + +impl NetSyncManager { + /// After a restart, proactively re-gossip this node's own already-produced forwardable DKG + /// artifacts (H3/H11). Resume from a persisted phase is otherwise passive: the restored + /// keyshare/aggregator actors wait for peer documents and never re-emit their own outputs, so + /// peers that missed the original gossip (cache expiry, DHT miss, peer churn) can stall the + /// node to its phase timeout. + /// + /// The artifacts are sent straight to libp2p as `GossipPublish`, bypassing both the EventBus + /// dedup window (which already tracked them during replay) and the translator (which is only + /// created on `EffectsEnabled`). Re-broadcasting the byte-identical original payload is + /// equivocation-safe (peers dedup by event id) and idempotent. The query is bounded to the + /// snapshot-cursor window so only the in-flight (un-delivered) artifacts are re-sent. + pub(in crate::actors::net_sync_manager) fn maybe_rebroadcast_own_artifacts( + &mut self, + ctx: &mut actix::Context, + ) { + if self.rebroadcast_started || !self.net_ready { + return; + } + let Some(since) = self.rebroadcast_since.clone() else { + return; + }; + self.rebroadcast_started = true; + + let id = CorrelationId::new(); + self.rebroadcast_query_ids.insert(id); + info!("NetSyncManager: querying own forwardable artifacts for post-restart re-broadcast"); + if let Err(e) = self.eventstore.try_send( + EventStoreQueryBy::::new(id, since, ctx.address().recipient()) + .with_filter(EventStoreFilter::Source(EventSource::Local)), + ) { + error!("Failed to query EventStore for re-broadcast: {e}"); + self.rebroadcast_query_ids.remove(&id); + self.rebroadcast_started = false; + } + } + + /// Re-gossip the node's own forwardable artifacts returned by the re-broadcast query. + pub(in crate::actors::net_sync_manager) fn handle_rebroadcast_response( + &mut self, + events: Vec, + ) { + let mut count = 0usize; + for event in events { + if !EventTranslationService::is_forwardable_event(&event) { + continue; + } + let data: GossipData = match event.try_into() { + Ok(data) => data, + Err(e) => { + warn!("Failed to convert own artifact to gossip data: {e}"); + continue; + } + }; + if let Err(e) = self.tx.try_send(NetCommand::GossipPublish { + topic: self.topic.clone(), + data, + correlation_id: CorrelationId::new(), + }) { + warn!("Failed to re-broadcast own artifact (channel full or closed): {e}"); + } else { + count += 1; + } + } + info!("NetSyncManager: re-broadcast {count} own forwardable artifact(s) after restart"); + } +} diff --git a/crates/net/src/network_sync/handlers.rs b/crates/net/src/network_sync/handlers.rs new file mode 100644 index 0000000000..19fdd3f394 --- /dev/null +++ b/crates/net/src/network_sync/handlers.rs @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Actix routing for local replay, remote sync requests, and readiness signals. + +use super::*; + +impl Actor for NetSyncManager { + type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } +} + +/// Event broadcast from event bus +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + // We are making a sync request of another node + if let InterfoldEventData::HistoricalNetSyncStart(data) = msg { + // Capture the snapshot-cursor map so we can bound the post-restart re-broadcast of our + // own forwardable artifacts to the in-flight window (H3/H11). + self.rebroadcast_since = Some(data.since.clone().into_iter().collect()); + self.maybe_rebroadcast_own_artifacts(ctx); + ctx.notify(TypedEvent::new(data, ec)) + } + } +} + +/// SyncRequest is called on start up to fetch remote events +impl Handler> for NetSyncManager { + type Result = ResponseFuture<()>; + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + info!("HISTORICAL_NET_SYNC_START"); + trap_fut( + EType::Net, + &self.bus.with_ec(msg.get_ctx()), + handle_sync_request_event( + self.tx.clone(), + self.rx.clone(), + msg, + ctx.address(), + !self.readiness_all_peers_dialed(), + ), + ) + } +} + +impl NetSyncManager { + fn readiness_all_peers_dialed(&self) -> bool { + // `handle_sync_request_event` waits for a connection only if we have not yet observed the + // AllPeersDialed signal. The readiness machine tracks this; mirror its view here. + self.readiness.all_peers_dialed() + } +} + +/// We have received the sync response from the remote peer +impl Handler> for NetSyncManager { + type Result = (); + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + trap(EType::Net, &self.bus.with_ec(msg.get_ctx()), || { + info!("SYNC REQUEST SUCCEEDED"); + let (msg, ctx) = msg.into_components(); + let response = msg.response; + self.bus.publish_from_remote_as_response( + HistoricalNetSyncEventsReceived { + events: response.events.to_vec(), + }, + response.ts, + ctx, + None, + EventSource::Net, + )?; + + Ok(()) + }); + } +} + +/// We have received a sync request from a remote peer +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: IncomingRequest, ctx: &mut Self::Context) -> Self::Result { + trap(EType::Net, &self.bus.clone(), || { + let IncomingRequest { peer, responder } = msg; + let fetch_request: FetchEventsSince = match responder.try_request_into() { + Ok(request) => request, + Err(error) => { + warn!(%peer, %error, "Rejecting malformed historical-sync request"); + responder.bad_request("malformed historical sync request")?; + return Ok(()); + } + }; + if fetch_request.limit() == 0 { + responder.bad_request("limit must be greater than 0")?; + return Ok(()); + } + if let Some(reason) = self.request_capacity_error(&peer) { + warn!( + %peer, + in_flight = self.requests.len(), + "Rejecting historical-sync request: {reason}" + ); + responder.bad_request(reason)?; + return Ok(()); + } + + let id = CorrelationId::new(); + let scan_limit = sync_scan_limit(fetch_request.limit()); + info!( + peer = %peer, + correlation_id = %id, + requested_limit = fetch_request.limit(), + scan_limit, + "Processing incoming historical-sync request" + ); + let query: HashMap = + HashMap::from([(fetch_request.aggregate_id(), fetch_request.since())]); + self.requests + .insert(id, PendingSyncRequest { peer, responder }); + let storage_query = + EventStoreQueryBy::::new(id, query, ctx.address().recipient()) + .with_limit(scan_limit as u64); + if let Err(error) = self.eventstore.try_send(storage_query) { + if let Some(pending) = self.requests.remove(&id) { + pending.responder.respond(ProtocolResponse::Error( + "historical sync storage unavailable".to_string(), + ))?; + } + warn!(%peer, correlation_id = %id, %error, "Failed to query EventStore for sync"); + return Ok(()); + } + ctx.run_later(INCOMING_SYNC_REQUEST_TIMEOUT, move |this, _| { + this.expire_sync_request(id); + }); + Ok(()) + }); + } +} + +/// Receive Events from EventStore +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: EventStoreQueryResponse, _: &mut Self::Context) -> Self::Result { + // Post-restart re-broadcast response (own forwardable artifacts) — handled separately from + // peer sync-request responses. + if self.rebroadcast_query_ids.remove(&msg.id()) { + self.handle_rebroadcast_response(msg.into_events()); + return; + } + trap(EType::Net, &self.bus.clone(), || { + info!("Received response from eventstore."); + let Some(pending) = self.requests.remove(&msg.id()) else { + bail!("responder not found for {}", msg.id()); + }; + + let fetch_request: FetchEventsSince = pending.responder.try_request_into()?; + match build_sync_batch(msg.into_events(), &fetch_request) { + SyncBatchOutcome::BadRequest(reason) => pending.responder.bad_request(reason)?, + SyncBatchOutcome::Batch(batch) => pending.responder.ok(batch)?, + } + + Ok(()) + }) + } +} + +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, msg: AllPeersDialed, ctx: &mut Self::Context) -> Self::Result { + info!( + "NetSyncManager: AllPeersDialed (connected={}, total={})", + msg.connected, msg.total + ); + let decision = self.readiness.on_all_peers_dialed(msg.connected, msg.total); + self.apply_readiness(decision, ctx); + } +} + +impl Handler for NetSyncManager { + type Result = (); + fn handle(&mut self, _: PeerConnected, ctx: &mut Self::Context) -> Self::Result { + let decision = self.readiness.on_peer_connected(); + if let ReadinessDecision::PublishReady = decision { + info!("NetSyncManager: first peer connected"); + } + self.apply_readiness(decision, ctx); + } +} + +#[derive(Message)] +#[rtype(result = "()")] +pub(super) struct AllPeersDialed { + pub(super) connected: usize, + pub(super) total: usize, +} + +#[derive(Message)] +#[rtype(result = "()")] +pub(super) struct PeerConnected; diff --git a/crates/net/src/network_sync/tests.rs b/crates/net/src/network_sync/tests.rs new file mode 100644 index 0000000000..d3e5852180 --- /dev/null +++ b/crates/net/src/network_sync/tests.rs @@ -0,0 +1,297 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use crate::{ + direct_responder::ChannelType, + events::{IncomingRequest, NetCommand}, +}; +use actix::{Actor, Context as ActixContext, Handler}; +use e3_ciphernode_builder::EventSystem; +use e3_events::{E3id, EventSource, InterfoldEvent, PlaintextAggregated, TestEvent, Unsequenced}; +use e3_utils::ArcBytes; +use tokio::sync::{broadcast, mpsc, mpsc::UnboundedSender}; + +/// Minimal EventStore stand-in so `NetSyncManager::new` can be constructed in tests; the +/// re-broadcast unit test drives `handle_rebroadcast_response` directly and never queries it. +struct NoopEventStore; +impl Actor for NoopEventStore { + type Context = ActixContext; +} +impl Handler> for NoopEventStore { + type Result = (); + fn handle(&mut self, _: EventStoreQueryBy, _: &mut Self::Context) {} +} + +struct RecordingEventStore { + queries: UnboundedSender>, +} + +impl Actor for RecordingEventStore { + type Context = ActixContext; +} + +impl Handler> for RecordingEventStore { + type Result = (); + fn handle(&mut self, msg: EventStoreQueryBy, _: &mut Self::Context) { + let _ = self.queries.send(msg.limit()); + // Intentionally retain no response. Tests exercise the manager's in-flight bounds. + } +} + +fn manager_with_recording_store( + query_tx: UnboundedSender>, +) -> (NetSyncManager, mpsc::Receiver) { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle().unwrap().enable("test"); + let (tx, rx) = mpsc::channel::(100); + let (_evt_tx, evt_rx) = broadcast::channel::(100); + let evt_rx = Arc::new(evt_rx); + let eventstore = RecordingEventStore { queries: query_tx } + .start() + .recipient(); + + ( + NetSyncManager::new(&bus, &tx, &evt_rx, eventstore, "my-topic"), + rx, + ) +} + +fn incoming_sync_request( + peer: PeerId, + id: u64, + limit: usize, + tx: &mpsc::Sender, +) -> IncomingRequest { + let request: Vec = FetchEventsSince::new(AggregateId::new(1), 0, limit) + .try_into() + .unwrap(); + let responder = DirectResponder::new(id, ChannelType::Test(format!("request-{id}")), tx) + .with_request(request); + IncomingRequest { peer, responder } +} + +fn protocol_response(command: NetCommand) -> ProtocolResponse { + let NetCommand::IncomingResponse(incoming) = command else { + panic!("expected IncomingResponse, got {command:?}"); + }; + incoming.responder.to_response().unwrap().1 +} + +fn local_forwardable_event(e3: &str) -> InterfoldEvent { + InterfoldEvent::::new_with_timestamp( + PlaintextAggregated { + e3_id: E3id::new(e3, 1), + decrypted_output: vec![ArcBytes::from_bytes(&[1, 2, 3, 4])], + decryption_aggregator_proofs: vec![], + } + .into(), + None, + 10, + None, + EventSource::Local, + ) + .into_sequenced(1) +} + +fn local_non_forwardable_event() -> InterfoldEvent { + InterfoldEvent::::new_with_timestamp( + TestEvent::new("not-forwardable", 1).into(), + None, + 11, + None, + EventSource::Local, + ) + .into_sequenced(2) +} + +fn remote_unsequenced(event: InterfoldEvent) -> InterfoldEvent { + event.clone_unsequenced().with_source(EventSource::Net) +} + +#[test] +fn historical_sync_rejects_non_forwardable_remote_events() { + let error = validate_historical_events( + AggregateId::new(0), + vec![remote_unsequenced(local_non_forwardable_event())], + ) + .unwrap_err(); + + assert!(error + .to_string() + .contains("non-forwardable event type TestEvent")); +} + +#[test] +fn historical_sync_rejects_events_from_another_aggregate() { + let event = remote_unsequenced(local_forwardable_event("1234")); + + let error = validate_historical_events(AggregateId::new(999), vec![event]).unwrap_err(); + + assert!(error.to_string().contains("while fetching 999")); +} + +#[actix::test] +async fn rebroadcast_only_gossips_forwardable_own_artifacts() { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle().unwrap().enable("test"); + let (tx, mut rx) = mpsc::channel::(100); + let (_evt_tx, evt_rx) = broadcast::channel::(100); + let evt_rx = Arc::new(evt_rx); + let eventstore = NoopEventStore.start().recipient(); + + let mut mgr = NetSyncManager::new(&bus, &tx, &evt_rx, eventstore, "my-topic"); + + mgr.handle_rebroadcast_response(vec![ + local_forwardable_event("1234"), + local_non_forwardable_event(), + ]); + + // Exactly one GossipPublish for the forwardable artifact, on the configured topic. + let cmd = rx.try_recv().expect("expected a GossipPublish command"); + let NetCommand::GossipPublish { topic, data, .. } = cmd else { + panic!("expected GossipPublish, got {cmd:?}"); + }; + assert_eq!(topic, "my-topic"); + let event: InterfoldEvent = data.try_into().unwrap(); + assert!(matches!( + event.get_data(), + InterfoldEventData::PlaintextAggregated(_) + )); + + // The non-forwardable event must not have produced a second command. + assert!( + rx.try_recv().is_err(), + "non-forwardable event should not be re-broadcast" + ); +} + +#[actix::test] +async fn malicious_huge_limit_is_capped_before_storage_query() { + let (query_tx, mut query_rx) = mpsc::unbounded_channel(); + let (manager, _net_rx) = manager_with_recording_store(query_tx); + let net_tx = manager.tx.clone(); + let manager = manager.start(); + + manager + .send(incoming_sync_request( + PeerId::random(), + 1, + usize::MAX, + &net_tx, + )) + .await + .unwrap(); + + let queried_limit = tokio::time::timeout(Duration::from_secs(1), query_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!(queried_limit, Some(sync_scan_limit(usize::MAX) as u64)); +} + +#[actix::test] +async fn concurrent_sync_requests_are_globally_bounded() { + let (query_tx, mut query_rx) = mpsc::unbounded_channel(); + let (manager, mut net_rx) = manager_with_recording_store(query_tx); + let net_tx = manager.tx.clone(); + let manager = manager.start(); + + for id in 0..MAX_IN_FLIGHT_SYNC_REQUESTS as u64 { + manager + .send(incoming_sync_request(PeerId::random(), id, 1, &net_tx)) + .await + .unwrap(); + } + manager + .send(incoming_sync_request( + PeerId::random(), + MAX_IN_FLIGHT_SYNC_REQUESTS as u64, + 1, + &net_tx, + )) + .await + .unwrap(); + + for _ in 0..MAX_IN_FLIGHT_SYNC_REQUESTS { + tokio::time::timeout(Duration::from_secs(1), query_rx.recv()) + .await + .unwrap() + .unwrap(); + } + assert!( + query_rx.try_recv().is_err(), + "overflow request reached storage" + ); + let response = tokio::time::timeout(Duration::from_secs(1), net_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!( + protocol_response(response), + ProtocolResponse::BadRequest(reason) if reason.contains("too many in-flight") + )); +} + +#[actix::test] +async fn concurrent_sync_requests_are_bounded_per_authenticated_peer() { + let (query_tx, mut query_rx) = mpsc::unbounded_channel(); + let (manager, mut net_rx) = manager_with_recording_store(query_tx); + let net_tx = manager.tx.clone(); + let manager = manager.start(); + let peer = PeerId::random(); + + for id in 0..=MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER as u64 { + manager + .send(incoming_sync_request(peer, id, 1, &net_tx)) + .await + .unwrap(); + } + + for _ in 0..MAX_IN_FLIGHT_SYNC_REQUESTS_PER_PEER { + tokio::time::timeout(Duration::from_secs(1), query_rx.recv()) + .await + .unwrap() + .unwrap(); + } + assert!( + query_rx.try_recv().is_err(), + "per-peer overflow reached storage" + ); + let response = tokio::time::timeout(Duration::from_secs(1), net_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!( + protocol_response(response), + ProtocolResponse::BadRequest(reason) if reason.contains("this peer") + )); +} + +#[actix::test] +async fn timed_out_sync_request_releases_its_in_flight_slot() { + let (query_tx, _query_rx) = mpsc::unbounded_channel(); + let (mut manager, mut net_rx) = manager_with_recording_store(query_tx); + let net_tx = manager.tx.clone(); + let peer = PeerId::random(); + let IncomingRequest { responder, .. } = incoming_sync_request(peer, 1, 1, &net_tx); + let id = CorrelationId::new(); + manager + .requests + .insert(id, PendingSyncRequest { peer, responder }); + + manager.expire_sync_request(id); + + assert!(manager.requests.is_empty()); + let response = tokio::time::timeout(Duration::from_secs(1), net_rx.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!( + protocol_response(response), + ProtocolResponse::Error(reason) if reason.contains("timed out") + )); +} diff --git a/crates/net/src/domain/wire.rs b/crates/net/src/network_sync/wire.rs similarity index 100% rename from crates/net/src/domain/wire.rs rename to crates/net/src/network_sync/wire.rs diff --git a/crates/net/src/domain/sync_coordinator.rs b/crates/net/src/network_sync/workflow.rs similarity index 50% rename from crates/net/src/domain/sync_coordinator.rs rename to crates/net/src/network_sync/workflow.rs index 5d5488ade7..867f053192 100644 --- a/crates/net/src/domain/sync_coordinator.rs +++ b/crates/net/src/network_sync/workflow.rs @@ -164,175 +164,5 @@ pub fn build_sync_batch( } #[cfg(test)] -mod tests { - use super::*; - use e3_events::{ - AggregateId, E3id, EventConstructorWithTimestamp, EventSource, PlaintextAggregated, - TestEvent, - }; - use e3_utils::ArcBytes; - - #[test] - fn no_peers_publishes_immediately_and_is_idempotent() { - let mut r = NetReadiness::new(); - assert_eq!(r.on_all_peers_dialed(0, 0), ReadinessDecision::PublishReady); - assert_eq!(r.on_all_peers_dialed(0, 0), ReadinessDecision::Idle); - } - - #[test] - fn connected_peers_publish_ready() { - let mut r = NetReadiness::new(); - assert_eq!(r.on_all_peers_dialed(2, 3), ReadinessDecision::PublishReady); - } - - #[test] - fn all_dials_failed_waits_then_publishes_on_connect() { - let mut r = NetReadiness::new(); - assert_eq!( - r.on_all_peers_dialed(0, 3), - ReadinessDecision::WaitForConnection - ); - assert_eq!(r.on_peer_connected(), ReadinessDecision::PublishReady); - assert_eq!(r.on_peer_connected(), ReadinessDecision::Idle); - } - - #[test] - fn timeout_publishes_when_no_connection_arrived() { - let mut r = NetReadiness::new(); - assert_eq!( - r.on_all_peers_dialed(0, 3), - ReadinessDecision::WaitForConnection - ); - assert_eq!(r.on_connect_timeout(), ReadinessDecision::PublishReady); - assert_eq!(r.on_connect_timeout(), ReadinessDecision::Idle); - } - - #[test] - fn peer_connected_before_dial_does_not_publish() { - let mut r = NetReadiness::new(); - assert_eq!(r.on_peer_connected(), ReadinessDecision::Idle); - // Once dialing finishes with the connection already present, publish. - assert_eq!(r.on_all_peers_dialed(0, 3), ReadinessDecision::PublishReady); - } - - fn net_event(ts: u128) -> InterfoldEvent { - InterfoldEvent::::new_with_timestamp( - PlaintextAggregated { - e3_id: E3id::new(ts.to_string(), 1), - decrypted_output: vec![ArcBytes::from_bytes(&[1, 2, 3])], - decryption_aggregator_proofs: vec![], - } - .into(), - None, - ts, - None, - EventSource::Net, - ) - .into_sequenced(ts as u64) - } - - fn net_non_forwardable_event(ts: u128) -> InterfoldEvent { - InterfoldEvent::::new_with_timestamp( - TestEvent::new("remote-control", ts as u64).into(), - None, - ts, - None, - EventSource::Net, - ) - .into_sequenced(ts as u64) - } - - fn local_event(ts: u128) -> InterfoldEvent { - InterfoldEvent::::new_with_timestamp( - TestEvent::new("y", ts as u64).into(), - None, - ts, - None, - EventSource::Local, - ) - .into_sequenced(ts as u64) - } - - #[test] - fn build_sync_batch_rejects_zero_limit() { - let fetch = FetchEventsSince::new(AggregateId::new(1), 0, 0); - assert!(matches!( - build_sync_batch(vec![], &fetch), - SyncBatchOutcome::BadRequest(_) - )); - } - - #[test] - fn build_sync_batch_filters_local_non_forwardable_and_marks_done() { - let fetch = FetchEventsSince::new(AggregateId::new(1), 0, 10); - let outcome = build_sync_batch( - vec![net_event(5), net_non_forwardable_event(6), local_event(7)], - &fetch, - ); - let SyncBatchOutcome::Batch(batch) = outcome else { - panic!("expected batch"); - }; - // Only the allowlisted protocol event survives. Remote source does not make an internal - // TestEvent forwardable. - assert_eq!(batch.events.len(), 1); - assert!(matches!(batch.next, BatchCursor::Done)); - } - - #[test] - fn build_sync_batch_limit_one_advances_past_inclusive_cursor() { - let fetch = FetchEventsSince::new(AggregateId::new(1), 0, 1); - let outcome = build_sync_batch(vec![net_event(5), net_event(9)], &fetch); - let SyncBatchOutcome::Batch(batch) = outcome else { - panic!("expected batch"); - }; - assert_eq!(batch.events.len(), 1); - assert!(matches!(batch.next, BatchCursor::Next(6))); - } - - #[test] - fn build_sync_batch_caps_malicious_huge_limit() { - let fetch = FetchEventsSince::new(AggregateId::new(1), 0, usize::MAX); - let events = (1..=MAX_SYNC_BATCH_SIZE + 1) - .map(|ts| net_event(ts as u128)) - .collect(); - let SyncBatchOutcome::Batch(batch) = build_sync_batch(events, &fetch) else { - panic!("expected batch"); - }; - - assert_eq!(batch.events.len(), MAX_SYNC_BATCH_SIZE); - assert!(matches!( - batch.next, - BatchCursor::Next(next) if next == MAX_SYNC_BATCH_SIZE as u128 + 1 - )); - assert_eq!(sync_scan_limit(fetch.limit()), MAX_SYNC_SCAN_EVENTS); - } - - #[test] - fn build_sync_batch_advances_past_full_filtered_scan() { - let fetch = FetchEventsSince::new(AggregateId::new(1), 0, 1); - let events = (1..=sync_scan_limit(fetch.limit())) - .map(|ts| local_event(ts as u128)) - .collect(); - let SyncBatchOutcome::Batch(batch) = build_sync_batch(events, &fetch) else { - panic!("expected batch"); - }; - - assert!(batch.events.is_empty()); - assert!(matches!( - batch.next, - BatchCursor::Next(next) if next == sync_scan_limit(fetch.limit()) as u128 + 1 - )); - } - - #[test] - fn build_sync_batch_stops_at_max_timestamp() { - let fetch = FetchEventsSince::new(AggregateId::new(1), u128::MAX, 1); - let SyncBatchOutcome::Batch(batch) = build_sync_batch(vec![net_event(u128::MAX)], &fetch) - else { - panic!("expected batch"); - }; - - assert_eq!(batch.events.len(), 1); - assert!(matches!(batch.next, BatchCursor::Done)); - } -} +#[path = "workflow_tests.rs"] +mod tests; diff --git a/crates/net/src/network_sync/workflow_tests.rs b/crates/net/src/network_sync/workflow_tests.rs new file mode 100644 index 0000000000..26a1885f83 --- /dev/null +++ b/crates/net/src/network_sync/workflow_tests.rs @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use e3_events::{ + AggregateId, E3id, EventConstructorWithTimestamp, EventSource, PlaintextAggregated, TestEvent, +}; +use e3_utils::ArcBytes; + +#[test] +fn no_peers_publishes_immediately_and_is_idempotent() { + let mut r = NetReadiness::new(); + assert_eq!(r.on_all_peers_dialed(0, 0), ReadinessDecision::PublishReady); + assert_eq!(r.on_all_peers_dialed(0, 0), ReadinessDecision::Idle); +} + +#[test] +fn connected_peers_publish_ready() { + let mut r = NetReadiness::new(); + assert_eq!(r.on_all_peers_dialed(2, 3), ReadinessDecision::PublishReady); +} + +#[test] +fn all_dials_failed_waits_then_publishes_on_connect() { + let mut r = NetReadiness::new(); + assert_eq!( + r.on_all_peers_dialed(0, 3), + ReadinessDecision::WaitForConnection + ); + assert_eq!(r.on_peer_connected(), ReadinessDecision::PublishReady); + assert_eq!(r.on_peer_connected(), ReadinessDecision::Idle); +} + +#[test] +fn timeout_publishes_when_no_connection_arrived() { + let mut r = NetReadiness::new(); + assert_eq!( + r.on_all_peers_dialed(0, 3), + ReadinessDecision::WaitForConnection + ); + assert_eq!(r.on_connect_timeout(), ReadinessDecision::PublishReady); + assert_eq!(r.on_connect_timeout(), ReadinessDecision::Idle); +} + +#[test] +fn peer_connected_before_dial_does_not_publish() { + let mut r = NetReadiness::new(); + assert_eq!(r.on_peer_connected(), ReadinessDecision::Idle); + // Once dialing finishes with the connection already present, publish. + assert_eq!(r.on_all_peers_dialed(0, 3), ReadinessDecision::PublishReady); +} + +fn net_event(ts: u128) -> InterfoldEvent { + InterfoldEvent::::new_with_timestamp( + PlaintextAggregated { + e3_id: E3id::new(ts.to_string(), 1), + decrypted_output: vec![ArcBytes::from_bytes(&[1, 2, 3])], + decryption_aggregator_proofs: vec![], + } + .into(), + None, + ts, + None, + EventSource::Net, + ) + .into_sequenced(ts as u64) +} + +fn net_non_forwardable_event(ts: u128) -> InterfoldEvent { + InterfoldEvent::::new_with_timestamp( + TestEvent::new("remote-control", ts as u64).into(), + None, + ts, + None, + EventSource::Net, + ) + .into_sequenced(ts as u64) +} + +fn local_event(ts: u128) -> InterfoldEvent { + InterfoldEvent::::new_with_timestamp( + TestEvent::new("y", ts as u64).into(), + None, + ts, + None, + EventSource::Local, + ) + .into_sequenced(ts as u64) +} + +#[test] +fn build_sync_batch_rejects_zero_limit() { + let fetch = FetchEventsSince::new(AggregateId::new(1), 0, 0); + assert!(matches!( + build_sync_batch(vec![], &fetch), + SyncBatchOutcome::BadRequest(_) + )); +} + +#[test] +fn build_sync_batch_filters_local_non_forwardable_and_marks_done() { + let fetch = FetchEventsSince::new(AggregateId::new(1), 0, 10); + let outcome = build_sync_batch( + vec![net_event(5), net_non_forwardable_event(6), local_event(7)], + &fetch, + ); + let SyncBatchOutcome::Batch(batch) = outcome else { + panic!("expected batch"); + }; + // Only the allowlisted protocol event survives. Remote source does not make an internal + // TestEvent forwardable. + assert_eq!(batch.events.len(), 1); + assert!(matches!(batch.next, BatchCursor::Done)); +} + +#[test] +fn build_sync_batch_limit_one_advances_past_inclusive_cursor() { + let fetch = FetchEventsSince::new(AggregateId::new(1), 0, 1); + let outcome = build_sync_batch(vec![net_event(5), net_event(9)], &fetch); + let SyncBatchOutcome::Batch(batch) = outcome else { + panic!("expected batch"); + }; + assert_eq!(batch.events.len(), 1); + assert!(matches!(batch.next, BatchCursor::Next(6))); +} + +#[test] +fn build_sync_batch_caps_malicious_huge_limit() { + let fetch = FetchEventsSince::new(AggregateId::new(1), 0, usize::MAX); + let events = (1..=MAX_SYNC_BATCH_SIZE + 1) + .map(|ts| net_event(ts as u128)) + .collect(); + let SyncBatchOutcome::Batch(batch) = build_sync_batch(events, &fetch) else { + panic!("expected batch"); + }; + + assert_eq!(batch.events.len(), MAX_SYNC_BATCH_SIZE); + assert!(matches!( + batch.next, + BatchCursor::Next(next) if next == MAX_SYNC_BATCH_SIZE as u128 + 1 + )); + assert_eq!(sync_scan_limit(fetch.limit()), MAX_SYNC_SCAN_EVENTS); +} + +#[test] +fn build_sync_batch_advances_past_full_filtered_scan() { + let fetch = FetchEventsSince::new(AggregateId::new(1), 0, 1); + let events = (1..=sync_scan_limit(fetch.limit())) + .map(|ts| local_event(ts as u128)) + .collect(); + let SyncBatchOutcome::Batch(batch) = build_sync_batch(events, &fetch) else { + panic!("expected batch"); + }; + + assert!(batch.events.is_empty()); + assert!(matches!( + batch.next, + BatchCursor::Next(next) if next == sync_scan_limit(fetch.limit()) as u128 + 1 + )); +} + +#[test] +fn build_sync_batch_stops_at_max_timestamp() { + let fetch = FetchEventsSince::new(AggregateId::new(1), u128::MAX, 1); + let SyncBatchOutcome::Batch(batch) = build_sync_batch(vec![net_event(u128::MAX)], &fetch) + else { + panic!("expected batch"); + }; + + assert_eq!(batch.events.len(), 1); + assert!(matches!(batch.next, BatchCursor::Done)); +} diff --git a/crates/net/src/domain/peer_failure_tracker.rs b/crates/net/src/peer_failure_tracker.rs similarity index 100% rename from crates/net/src/domain/peer_failure_tracker.rs rename to crates/net/src/peer_failure_tracker.rs diff --git a/crates/request/src/actors/mod.rs b/crates/request/src/actors.rs similarity index 65% rename from crates/request/src/actors/mod.rs rename to crates/request/src/actors.rs index 64eb239582..3ec1c5c6b8 100644 --- a/crates/request/src/actors/mod.rs +++ b/crates/request/src/actors.rs @@ -4,7 +4,11 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +//! Compatibility view of actors stored with the routing and lifecycle capabilities. + +#[path = "lifecycle/actor.rs"] mod lifecycle_coordinator; +#[path = "routing/actor.rs"] mod router; pub use lifecycle_coordinator::*; diff --git a/crates/request/src/actors/router.rs b/crates/request/src/actors/router.rs deleted file mode 100644 index 6246caa44d..0000000000 --- a/crates/request/src/actors/router.rs +++ /dev/null @@ -1,389 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::ContextRepositoryFactory; -use crate::E3Context; -use crate::E3ContextParams; -use crate::E3MetaExtension; -use crate::EventBuffer; -use crate::PostForward; -use crate::RequestRouter; -use crate::RouterRepositoryFactory; -use crate::RoutingDecision; -use actix::{Actor, Addr, Context, Handler}; -use anyhow::*; -use async_trait::async_trait; -use e3_data::Checkpoint; -use e3_data::DataStore; -use e3_data::FromSnapshotWithParams; -use e3_data::RepositoriesFactory; -use e3_data::Repository; -use e3_data::Snapshot; -use e3_events::prelude::*; -use e3_events::trap; -use e3_events::BusHandle; -use e3_events::E3RequestComplete; -use e3_events::EType; -use e3_events::EventType; -use e3_events::{E3id, InterfoldEvent}; -use e3_utils::MAILBOX_LIMIT; -use serde::Deserialize; -use serde::Serialize; -use std::collections::HashSet; -use std::{collections::HashMap, sync::Arc}; - -/// An Extension interface for the E3Router system that listens and responds to InterfoldEvents. -/// -/// # Responsibilities -/// - Listens for broadcast InterfoldEvents -/// - Instantiates appropriate actors based on received events -/// - Manages actor state persistence and reconstruction -/// - Handles event streaming to registered recipients -/// -/// # Usage -/// Extensions implement the `on_event` handler to define which events they respond to. -/// When an event is received, the extension typically: -/// 1. Uses the request's context to construct required actors -/// 2. Saves actor addresses to the context using `set_event_recipient` -/// 3. Manages event streaming from buffers to registered recipients -/// -/// Extensions can also reconstruct actors from persisted state using context -/// snapshots and repositories. They can check for dependencies in the context -/// before constructing new extensions. -#[async_trait] -pub trait E3Extension: Send + Sync + 'static { - /// This function is triggered when an InterfoldEvent is sent to the router. Use this to - /// initialize the receiver using `ctx.set_event_receiver(my_address.into())`. Typically this - /// means filtering for specific e3_id enabled events that give rise to actors that have to - /// handle certain behaviour. - fn on_event(&self, ctx: &mut E3Context, evt: &InterfoldEvent); - - /// This function it triggered when the request context is being hydrated from snapshot. - async fn hydrate(&self, ctx: &mut E3Context, snapshot: &crate::E3ContextSnapshot) - -> Result<()>; -} - -/// Routes E3_id-specific contexts to registered extensions and manages message forwarding. -/// -/// # Core Functions -/// - Maintains contexts for each E3 request -/// - Lazily registers extension instances with appropriate dependencies per E3_id -/// - Forwards incoming messages to registered instances -/// - Manages request lifecycle and completion -/// -/// Extensions receive an E3_id-specific context and can handle specific -/// message types. The router ensures proper message delivery and context management -/// throughout the request lifecycle. -/// -/// This actor is a thin message-passing shell: all routing decisions are computed by the -/// pure [`RequestRouter`] service; the actor only performs the resulting actix I/O. -// TODO: setup so that we have to place extensions within correct order of dependencies -pub struct E3Router { - /// The context for every E3 request - contexts: HashMap, - /// A list of completed requests - completed: HashSet, - /// The extensions this instance of the router is configured to listen for - extensions: Arc>>, - /// A buffer for events to send to the - buffer: EventBuffer, - /// The EventBus - bus: BusHandle, - /// A repository for storing snapshots - store: Repository, -} - -pub struct E3RouterParams { - extensions: Arc>>, - bus: BusHandle, - store: Repository, -} - -impl E3Router { - pub fn builder(bus: &BusHandle, store: DataStore) -> E3RouterBuilder { - let repositories = store.repositories(); - let builder = E3RouterBuilder { - bus: bus.clone(), - extensions: vec![], - store: repositories.router(), - }; - - // Everything needs the committe meta factory so adding it here by default - builder.with(E3MetaExtension::create()) - } - - pub fn from_params(params: E3RouterParams) -> Self { - Self { - extensions: params.extensions, - bus: params.bus.clone(), - store: params.store.clone(), - completed: HashSet::new(), - contexts: HashMap::new(), - buffer: EventBuffer::default(), - } - } -} - -impl Actor for E3Router { - type Context = Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT) - } -} - -impl Handler for E3Router { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { - trap(EType::Event, &self.bus.with_ec(msg.get_ctx()), || { - match RequestRouter::route(&msg, &self.completed) { - RoutingDecision::Broadcast => { - // If we are shutting down then bail on anything else - for (_, ctx) in self.contexts.iter() { - ctx.forward_message_now(&msg) - } - Ok(()) - } - RoutingDecision::Ignore => Ok(()), - RoutingDecision::AlreadyCompleted(e3_id) => Err(anyhow!( - "unexpected {} for completed E3 {} (event={}, origin={}, source={:?}, block={:?})", - msg.event_type(), - e3_id, - msg.id(), - msg.origin_id(), - msg.source(), - msg.block(), - )), - RoutingDecision::Process { - e3_id, - post_forward, - } => { - let repositories = self.repository().repositories(); - let context = self.contexts.entry(e3_id.clone()).or_insert_with(|| { - E3Context::from_params(E3ContextParams { - e3_id: e3_id.clone(), - repository: repositories.context(&e3_id), - extensions: self.extensions.clone(), - }) - }); - - for extension in self.extensions.iter() { - extension.on_event(context, &msg); - } - - context.forward_message(&msg, &mut self.buffer); - - let (_, ctx) = msg.into_components(); - match post_forward { - PostForward::PublishComplete => { - let event = E3RequestComplete { - e3_id: e3_id.clone(), - }; - - // Send to bus so all other actors can react to a request being complete. - self.bus.publish(event, ctx)?; - } - PostForward::Teardown => { - // The original event is forwarded above so children can kill themselves. - self.contexts.remove(&e3_id); - self.buffer.remove_e3(&e3_id); - self.completed.insert(e3_id); - } - PostForward::None => (), - } - - self.checkpoint(); - Ok(()) - } - } - }); - } -} - -#[derive(Serialize, Deserialize)] -pub struct E3RouterSnapshot { - contexts: Vec, - completed: HashSet, -} - -impl Snapshot for E3Router { - type Snapshot = E3RouterSnapshot; - fn snapshot(&self) -> Result { - let contexts = self.contexts.keys().cloned().collect(); - let completed = self.completed.clone(); - - Ok(Self::Snapshot { - completed, - contexts, - }) - } -} - -impl Checkpoint for E3Router { - fn repository(&self) -> &Repository { - &self.store - } -} - -#[async_trait] -impl FromSnapshotWithParams for E3Router { - type Params = E3RouterParams; - - async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { - let mut contexts = HashMap::new(); - - let repositories = params.store.repositories(); - for e3_id in snapshot.contexts { - let Some(ctx_snapshot) = repositories.context(&e3_id).read().await? else { - continue; - }; - - contexts.insert( - e3_id.clone(), - E3Context::from_snapshot( - E3ContextParams { - repository: repositories.context(&e3_id), - e3_id: e3_id.clone(), - extensions: params.extensions.clone(), - }, - ctx_snapshot, - ) - .await?, - ); - } - - Ok(E3Router { - contexts, - completed: snapshot.completed, - extensions: params.extensions, - buffer: EventBuffer::default(), - bus: params.bus, - store: params.store, - }) - } -} - -/// Builder for E3Router -pub struct E3RouterBuilder { - pub bus: BusHandle, - pub extensions: Vec>, - pub store: Repository, -} - -impl E3RouterBuilder { - pub fn with(mut self, listener: Box) -> Self { - self.extensions.push(listener); - self - } - - pub async fn build(self) -> Result> { - let snapshot: Option = self.store.read().await?; - let params = E3RouterParams { - extensions: self.extensions.into(), - bus: self.bus.clone(), - store: self.store.clone(), - }; - - let e3r = match snapshot { - Some(snapshot) => E3Router::from_snapshot(params, snapshot).await?, - None => E3Router::from_params(params), - }; - - let addr = e3r.start(); - self.bus.subscribe(EventType::All, addr.clone().recipient()); - Ok(addr) - } -} - -#[cfg(test)] -mod recovery_tests { - use super::*; - use crate::E3ContextSnapshot; - use actix::{Actor, Handler}; - use async_trait::async_trait; - use e3_data::{InMemStore, RepositoriesFactory}; - use e3_events::{hlc_factory::HlcFactory, BusHandle, EventBus, Sequencer, StoreEventRequested}; - use std::sync::{ - atomic::{AtomicUsize, Ordering}, - Arc, - }; - - struct StoreSink; - - impl Actor for StoreSink { - type Context = Context; - } - - impl Handler for StoreSink { - type Result = (); - - fn handle(&mut self, _: StoreEventRequested, _: &mut Self::Context) {} - } - - struct RecoveryExtension { - hydrations: Arc, - } - - #[async_trait] - impl E3Extension for RecoveryExtension { - fn on_event(&self, _: &mut E3Context, _: &InterfoldEvent) {} - - async fn hydrate(&self, _: &mut E3Context, _: &E3ContextSnapshot) -> Result<()> { - self.hydrations.fetch_add(1, Ordering::SeqCst); - Ok(()) - } - } - - fn test_bus() -> BusHandle { - let event_bus = EventBus::::default().start(); - let store = StoreSink.start(); - let sequencer = Sequencer::new(&event_bus, store.recipient()).start(); - BusHandle::new(event_bus, sequencer, HlcFactory::new()).enable("router-recovery-test") - } - - #[actix::test] - async fn mid_e3_context_and_completed_set_survive_hydration() -> Result<()> { - let active = E3id::new("7", 31337); - let complete = E3id::new("6", 31337); - let store = DataStore::from_in_mem(&InMemStore::new(false).start()); - let repositories = store.repositories(); - let router_store = repositories.router(); - router_store - .repositories() - .context(&active) - .write_sync(&E3ContextSnapshot { - e3_id: active.clone(), - recipients: vec!["threshold_keyshare".into()], - dependencies: vec!["meta".into()], - }) - .await?; - - let hydrations = Arc::new(AtomicUsize::new(0)); - let params = E3RouterParams { - extensions: Arc::new(vec![Box::new(RecoveryExtension { - hydrations: hydrations.clone(), - })]), - bus: test_bus(), - store: router_store, - }; - let recovered = E3Router::from_snapshot( - params, - E3RouterSnapshot { - contexts: vec![active.clone()], - completed: HashSet::from([complete.clone()]), - }, - ) - .await?; - - assert!(recovered.contexts.contains_key(&active)); - assert!(recovered.completed.contains(&complete)); - assert_eq!(hydrations.load(Ordering::SeqCst), 1); - - let roundtrip = recovered.snapshot()?; - assert_eq!(roundtrip.contexts, vec![active]); - assert_eq!(roundtrip.completed, HashSet::from([complete])); - Ok(()) - } -} diff --git a/crates/request/src/domain/mod.rs b/crates/request/src/domain.rs similarity index 63% rename from crates/request/src/domain/mod.rs rename to crates/request/src/domain.rs index 4fdc9de074..f806852a69 100644 --- a/crates/request/src/domain/mod.rs +++ b/crates/request/src/domain.rs @@ -4,8 +4,13 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +//! Compatibility view of pure request modules stored by capability. + +#[path = "routing/event_buffer.rs"] mod event_buffer; +#[path = "lifecycle/workflow.rs"] mod lifecycle; +#[path = "routing/workflow.rs"] mod routing; pub use event_buffer::*; diff --git a/crates/request/src/domain/routing.rs b/crates/request/src/domain/routing.rs deleted file mode 100644 index 87c224be5d..0000000000 --- a/crates/request/src/domain/routing.rs +++ /dev/null @@ -1,444 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use e3_events::{E3Stage, E3id, Event, InterfoldEvent, InterfoldEventData}; -use std::collections::HashSet; - -/// The completion action a router should perform *after* running extension hooks and -/// forwarding an event to the request's context. -#[derive(Debug, PartialEq, Eq)] -pub enum PostForward { - /// Publish an `E3RequestComplete` event to the bus: the request has finished. - PublishComplete, - /// Tear down the context for this request and mark it as completed. - Teardown, - /// No completion action is required. - None, -} - -/// The decision the router makes when an event arrives, computed without performing any -/// actix message passing, persistence, or context mutation. -#[derive(Debug, PartialEq, Eq)] -pub enum RoutingDecision { - /// A shutdown event: broadcast it immediately to every active context. - Broadcast, - /// The event carries no `e3_id` and should be ignored. - Ignore, - /// The event targets a request that has already completed; this is an error. - AlreadyCompleted(E3id), - /// Process the event for the given request, applying `post_forward` after forwarding. - Process { - e3_id: E3id, - post_forward: PostForward, - }, -} - -/// Pure routing logic for the E3 request router. -/// -/// Classifies an incoming [`InterfoldEvent`] into a [`RoutingDecision`] based purely on the -/// event data and the set of already-completed requests. This contains no actix, persistence -/// or I/O concerns so it can be unit tested in isolation; the actor executes the decision. -pub struct RequestRouter; - -impl RequestRouter { - /// Decide how an incoming event should be routed given the set of completed requests. - pub fn route(msg: &InterfoldEvent, completed: &HashSet) -> RoutingDecision { - // Broadcast non-E3-scoped lifecycle signals to every active context: - // * `Shutdown` so children can tear themselves down, and - // * `EffectsEnabled` so a hydrated request can re-drive its own in-flight work - // once side effects are switched on at the end of boot sync. - // Both carry no `e3_id`, so without this they would be `Ignore`d and never reach the - // per-E3 child actors. - if matches!( - msg.get_data(), - InterfoldEventData::Shutdown(_) | InterfoldEventData::EffectsEnabled(_) - ) { - return RoutingDecision::Broadcast; - } - - // Durable observational EVM facts are consumed directly by projections - // and global observers. They describe an E3, but they do not drive its - // per-E3 actors. Routing them into a context would create contexts for - // historical observations and report expected post-settlement receipts - // as errors after teardown. - if matches!( - msg.get_data(), - InterfoldEventData::InputPublished(_) - | InterfoldEventData::RewardsDistributed(_) - | InterfoldEventData::RewardCredited(_) - | InterfoldEventData::RewardClaimed(_) - | InterfoldEventData::CommitteeFormationFailed(_) - | InterfoldEventData::CommitteeActivationChanged(_) - | InterfoldEventData::CommitteeViabilityUpdated(_) - | InterfoldEventData::EvmLogObserved(_) - ) { - return RoutingDecision::Ignore; - } - - // Only process events with e3_ids. - let Some(e3_id) = msg.get_e3_id() else { - return RoutingDecision::Ignore; - }; - - // If this e3 round has already been completed then this event is unexpected. - if completed.contains(&e3_id) { - // On-chain confirmation events that lag behind local teardown are expected and - // should be silently ignored rather than treated as an error. - let is_late_terminal = match msg.get_data() { - // E3StageChanged(Complete) always lags local PlaintextAggregated completion. - InterfoldEventData::E3StageChanged(data) - if matches!(data.new_stage, E3Stage::Complete | E3Stage::Failed) => - { - true - } - // E3Failed from on-chain markE3Failed may arrive after a local timeout already - // cleaned up the context. - InterfoldEventData::E3Failed(data) if data.reason.is_timeout() => true, - // Settlement receipts (PlaintextOutputPublished, etc.) can arrive in the same - // EVM block after E3StageChanged(Complete) already tore down the context. - InterfoldEventData::PlaintextOutputPublished(_) => true, - _ => false, - }; - if is_late_terminal { - return RoutingDecision::Ignore; - } - return RoutingDecision::AlreadyCompleted(e3_id); - } - - let post_forward = match msg.get_data() { - // Receiving the PlaintextAggregated event means the request is complete and we can - // notify everyone. This might change as we consider other completion factors. - InterfoldEventData::PlaintextAggregated(_) => PostForward::PublishComplete, - InterfoldEventData::E3StageChanged(data) - if matches!(data.new_stage, E3Stage::Complete) => - { - PostForward::PublishComplete - } - // Timeout failures have no accusation/slashing lifecycle, so the context can be - // torn down immediately. Misbehaviour failures (DKGInvalidShares, etc.) still need - // the accusation/slashing lifecycle to complete before teardown. - InterfoldEventData::E3Failed(data) if data.reason.is_timeout() => { - PostForward::PublishComplete - } - InterfoldEventData::E3RequestComplete(_) => PostForward::Teardown, - _ => PostForward::None, - }; - - RoutingDecision::Process { - e3_id, - post_forward, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use e3_events::{ - E3Failed, E3RequestComplete, E3Stage, E3StageChanged, FailureReason, InterfoldEvent, - PlaintextAggregated, RewardCredited, Sequenced, Shutdown, - }; - - fn e3id() -> E3id { - E3id::new("1", 1) - } - - fn with_e3_id(label: &str, id: E3id) -> InterfoldEvent { - InterfoldEvent::::test_event(label) - .e3_id(id) - .seq(1) - .build() - } - - fn from_data(data: impl Into) -> InterfoldEvent { - InterfoldEvent::::test_event("x") - .data(data) - .seq(1) - .build() - } - - #[test] - fn shutdown_broadcasts() { - let msg = from_data(Shutdown); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Broadcast - ); - } - - #[test] - fn effects_enabled_broadcasts() { - // EffectsEnabled has no e3_id but must reach every hydrated context so each can - // re-drive its own in-flight work after a restart. - let msg = from_data(e3_events::EffectsEnabled::new()); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Broadcast - ); - } - - #[test] - fn event_without_e3_id_is_ignored() { - let msg = InterfoldEvent::::test_event("no-id") - .seq(1) - .build(); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Ignore - ); - } - - #[test] - fn completed_request_is_an_error() { - let id = e3id(); - let mut completed = HashSet::new(); - completed.insert(id.clone()); - let msg = with_e3_id("late", id.clone()); - assert_eq!( - RequestRouter::route(&msg, &completed), - RoutingDecision::AlreadyCompleted(id) - ); - } - - #[test] - fn settlement_receipt_is_not_routed_to_completed_context() { - let id = e3id(); - let mut completed = HashSet::new(); - completed.insert(id.clone()); - let msg = from_data(RewardCredited { - e3_id: id, - account: "0x01".into(), - token: "0x02".into(), - amount: "10".into(), - }); - - assert_eq!( - RequestRouter::route(&msg, &completed), - RoutingDecision::Ignore - ); - } - - #[test] - fn stage_changed_to_complete_ignored_when_already_completed() { - // E3StageChanged(Complete) arriving from the EVM after local teardown is expected — - // the on-chain confirmation lags behind local completion. It should be silently - // ignored, not treated as an error. - let id = e3id(); - let mut completed = HashSet::new(); - completed.insert(id.clone()); - let msg = from_data(E3StageChanged { - e3_id: id.clone(), - previous_stage: E3Stage::CiphertextReady, - new_stage: E3Stage::Complete, - }); - assert_eq!( - RequestRouter::route(&msg, &completed), - RoutingDecision::Ignore - ); - } - - #[test] - fn stage_changed_to_failed_ignored_when_completed() { - // E3StageChanged(Failed) from the EVM can arrive after a local timeout already cleaned up - // the context. Treat it as a silent no-op, the same way we handle E3StageChanged(Complete). - let id = e3id(); - let mut completed = HashSet::new(); - completed.insert(id.clone()); - let msg = from_data(E3StageChanged { - e3_id: id.clone(), - previous_stage: E3Stage::CiphertextReady, - new_stage: E3Stage::Failed, - }); - assert_eq!( - RequestRouter::route(&msg, &completed), - RoutingDecision::Ignore - ); - } - - #[test] - fn plaintext_aggregated_publishes_complete() { - let id = e3id(); - let msg = from_data(PlaintextAggregated { - e3_id: id.clone(), - decrypted_output: vec![], - decryption_aggregator_proofs: vec![], - }); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Process { - e3_id: id, - post_forward: PostForward::PublishComplete, - } - ); - } - - #[test] - fn stage_changed_to_complete_publishes_complete() { - let id = e3id(); - let msg = from_data(E3StageChanged { - e3_id: id.clone(), - previous_stage: E3Stage::CiphertextReady, - new_stage: E3Stage::Complete, - }); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Process { - e3_id: id, - post_forward: PostForward::PublishComplete, - } - ); - } - - #[test] - fn stage_changed_to_failed_does_not_complete() { - let id = e3id(); - let msg = from_data(E3StageChanged { - e3_id: id.clone(), - previous_stage: E3Stage::CiphertextReady, - new_stage: E3Stage::Failed, - }); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Process { - e3_id: id, - post_forward: PostForward::None, - } - ); - } - - #[test] - fn e3_request_complete_triggers_teardown() { - // InterfoldEventData::get_e3_id() now returns Some(e3_id) for E3RequestComplete, - // so the event reaches the Teardown arm of the router. - let id = e3id(); - let msg = from_data(E3RequestComplete { e3_id: id.clone() }); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Process { - e3_id: id, - post_forward: PostForward::Teardown, - } - ); - } - - #[test] - fn generic_event_with_e3_id_has_no_completion() { - let id = e3id(); - let msg = with_e3_id("generic", id.clone()); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Process { - e3_id: id, - post_forward: PostForward::None, - } - ); - } - - // --- timeout-triggered E3Failed tests --- - - fn e3_failed(id: E3id, reason: FailureReason) -> InterfoldEvent { - from_data(E3Failed { - e3_id: id, - failed_at_stage: E3Stage::CommitteeFinalized, - reason, - }) - } - - #[test] - fn e3_failed_dkg_timeout_publishes_complete() { - let id = e3id(); - let msg = e3_failed(id.clone(), FailureReason::DKGTimeout); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Process { - e3_id: id, - post_forward: PostForward::PublishComplete, - } - ); - } - - #[test] - fn e3_failed_committee_formation_timeout_publishes_complete() { - let id = e3id(); - let msg = e3_failed(id.clone(), FailureReason::CommitteeFormationTimeout); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Process { - e3_id: id, - post_forward: PostForward::PublishComplete, - } - ); - } - - #[test] - fn e3_failed_compute_timeout_publishes_complete() { - let id = e3id(); - let msg = e3_failed(id.clone(), FailureReason::ComputeTimeout); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Process { - e3_id: id, - post_forward: PostForward::PublishComplete, - } - ); - } - - #[test] - fn e3_failed_decryption_timeout_publishes_complete() { - let id = e3id(); - let msg = e3_failed(id.clone(), FailureReason::DecryptionTimeout); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Process { - e3_id: id, - post_forward: PostForward::PublishComplete, - } - ); - } - - #[test] - fn e3_failed_invalid_shares_does_not_complete() { - // Slashable failures must NOT trigger E3RequestComplete — the accusation/slashing - // lifecycle must be allowed to finish first. - let id = e3id(); - let msg = e3_failed(id.clone(), FailureReason::DKGInvalidShares); - assert_eq!( - RequestRouter::route(&msg, &HashSet::new()), - RoutingDecision::Process { - e3_id: id, - post_forward: PostForward::None, - } - ); - } - - #[test] - fn e3_failed_timeout_ignored_when_already_completed() { - let id = e3id(); - let mut completed = HashSet::new(); - completed.insert(id.clone()); - let msg = e3_failed(id.clone(), FailureReason::DKGTimeout); - assert_eq!( - RequestRouter::route(&msg, &completed), - RoutingDecision::Ignore - ); - } - - #[test] - fn stage_changed_to_failed_ignored_when_already_completed() { - let id = e3id(); - let mut completed = HashSet::new(); - completed.insert(id.clone()); - let msg = from_data(E3StageChanged { - e3_id: id.clone(), - previous_stage: E3Stage::CommitteeFinalized, - new_stage: E3Stage::Failed, - }); - assert_eq!( - RequestRouter::route(&msg, &completed), - RoutingDecision::Ignore - ); - } -} diff --git a/crates/request/src/actors/lifecycle_coordinator.rs b/crates/request/src/lifecycle/actor.rs similarity index 100% rename from crates/request/src/actors/lifecycle_coordinator.rs rename to crates/request/src/lifecycle/actor.rs diff --git a/crates/request/src/domain/lifecycle.rs b/crates/request/src/lifecycle/workflow.rs similarity index 57% rename from crates/request/src/domain/lifecycle.rs rename to crates/request/src/lifecycle/workflow.rs index 51aa697f04..e485c12629 100644 --- a/crates/request/src/domain/lifecycle.rs +++ b/crates/request/src/lifecycle/workflow.rs @@ -171,157 +171,5 @@ impl E3LifecycleService { } #[cfg(test)] -mod tests { - use super::*; - use e3_events::{E3Failed, E3Requested, E3StageChanged, FailureReason}; - - fn id(n: &str) -> E3id { - E3id::new(n, 1) - } - - fn requested(n: &str) -> InterfoldEventData { - InterfoldEventData::E3Requested(E3Requested { - e3_id: id(n), - ..Default::default() - }) - } - - fn stage_changed(n: &str, from: E3Stage, to: E3Stage) -> InterfoldEventData { - InterfoldEventData::E3StageChanged(E3StageChanged { - e3_id: id(n), - previous_stage: from, - new_stage: to, - }) - } - - fn failed(n: &str, stage: E3Stage) -> InterfoldEventData { - InterfoldEventData::E3Failed(E3Failed { - e3_id: id(n), - failed_at_stage: stage, - reason: FailureReason::DKGTimeout, - }) - } - - #[test] - fn unknown_e3_is_stage_none() { - let svc = E3LifecycleService::new(); - assert_eq!(E3Stage::None, svc.stage(&id("x"))); - } - - #[test] - fn requested_advances_from_none() { - let mut svc = E3LifecycleService::new(); - let decision = svc.observe(&requested("a")); - assert_eq!( - LifecycleDecision::Advanced { - e3_id: id("a"), - from: E3Stage::None, - to: E3Stage::Requested - }, - decision - ); - assert_eq!(E3Stage::Requested, svc.stage(&id("a"))); - } - - #[test] - fn stage_advances_monotonically() { - let mut svc = E3LifecycleService::new(); - svc.observe(&requested("a")); - let d = svc.observe(&stage_changed( - "a", - E3Stage::Requested, - E3Stage::KeyPublished, - )); - assert!(matches!(d, LifecycleDecision::Advanced { .. })); - assert_eq!(E3Stage::KeyPublished, svc.stage(&id("a"))); - } - - #[test] - fn out_of_order_earlier_stage_is_regressed_and_ignored() { - let mut svc = E3LifecycleService::new(); - svc.observe(&stage_changed("a", E3Stage::None, E3Stage::KeyPublished)); - let d = svc.observe(&requested("a")); - assert_eq!( - LifecycleDecision::Regressed { - e3_id: id("a"), - current: E3Stage::KeyPublished, - attempted: E3Stage::Requested - }, - d - ); - // Tracked stage is unchanged. - assert_eq!(E3Stage::KeyPublished, svc.stage(&id("a"))); - } - - #[test] - fn same_stage_is_unchanged() { - let mut svc = E3LifecycleService::new(); - svc.observe(&requested("a")); - let d = svc.observe(&requested("a")); - assert_eq!( - LifecycleDecision::Unchanged { - e3_id: id("a"), - stage: E3Stage::Requested - }, - d - ); - } - - #[test] - fn failure_is_terminal_and_frozen() { - let mut svc = E3LifecycleService::new(); - svc.observe(&requested("a")); - let d = svc.observe(&failed("a", E3Stage::Requested)); - assert_eq!( - LifecycleDecision::Terminal { - e3_id: id("a"), - stage: E3Stage::Failed - }, - d - ); - // Further lifecycle events do not move a terminal E3. - let d2 = svc.observe(&stage_changed("a", E3Stage::Failed, E3Stage::Complete)); - assert_eq!( - LifecycleDecision::Unchanged { - e3_id: id("a"), - stage: E3Stage::Failed - }, - d2 - ); - assert_eq!(E3Stage::Failed, svc.stage(&id("a"))); - } - - #[test] - fn active_excludes_terminal_e3s() { - let mut svc = E3LifecycleService::new(); - svc.observe(&requested("a")); - svc.observe(&requested("b")); - svc.observe(&failed("b", E3Stage::Requested)); - let active = svc.active(); - assert_eq!(vec![id("a")], active); - } - - #[test] - fn non_lifecycle_event_is_ignored() { - let mut svc = E3LifecycleService::new(); - let d = svc.observe(&InterfoldEventData::Shutdown(e3_events::Shutdown)); - assert_eq!(LifecycleDecision::NotLifecycle, d); - } - - #[test] - fn snapshot_roundtrip_preserves_stages() { - let mut svc = E3LifecycleService::new(); - svc.observe(&requested("a")); - svc.observe(&stage_changed( - "a", - E3Stage::Requested, - E3Stage::CiphertextReady, - )); - svc.observe(&requested("b")); - - let snap = svc.snapshot(); - let restored = E3LifecycleService::from_snapshot(snap); - assert_eq!(E3Stage::CiphertextReady, restored.stage(&id("a"))); - assert_eq!(E3Stage::Requested, restored.stage(&id("b"))); - } -} +#[path = "workflow_tests.rs"] +mod tests; diff --git a/crates/request/src/lifecycle/workflow_tests.rs b/crates/request/src/lifecycle/workflow_tests.rs new file mode 100644 index 0000000000..77cda3fd9b --- /dev/null +++ b/crates/request/src/lifecycle/workflow_tests.rs @@ -0,0 +1,158 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use e3_events::{E3Failed, E3Requested, E3StageChanged, FailureReason}; + +fn id(n: &str) -> E3id { + E3id::new(n, 1) +} + +fn requested(n: &str) -> InterfoldEventData { + InterfoldEventData::E3Requested(E3Requested { + e3_id: id(n), + ..Default::default() + }) +} + +fn stage_changed(n: &str, from: E3Stage, to: E3Stage) -> InterfoldEventData { + InterfoldEventData::E3StageChanged(E3StageChanged { + e3_id: id(n), + previous_stage: from, + new_stage: to, + }) +} + +fn failed(n: &str, stage: E3Stage) -> InterfoldEventData { + InterfoldEventData::E3Failed(E3Failed { + e3_id: id(n), + failed_at_stage: stage, + reason: FailureReason::DKGTimeout, + }) +} + +#[test] +fn unknown_e3_is_stage_none() { + let svc = E3LifecycleService::new(); + assert_eq!(E3Stage::None, svc.stage(&id("x"))); +} + +#[test] +fn requested_advances_from_none() { + let mut svc = E3LifecycleService::new(); + let decision = svc.observe(&requested("a")); + assert_eq!( + LifecycleDecision::Advanced { + e3_id: id("a"), + from: E3Stage::None, + to: E3Stage::Requested + }, + decision + ); + assert_eq!(E3Stage::Requested, svc.stage(&id("a"))); +} + +#[test] +fn stage_advances_monotonically() { + let mut svc = E3LifecycleService::new(); + svc.observe(&requested("a")); + let d = svc.observe(&stage_changed( + "a", + E3Stage::Requested, + E3Stage::KeyPublished, + )); + assert!(matches!(d, LifecycleDecision::Advanced { .. })); + assert_eq!(E3Stage::KeyPublished, svc.stage(&id("a"))); +} + +#[test] +fn out_of_order_earlier_stage_is_regressed_and_ignored() { + let mut svc = E3LifecycleService::new(); + svc.observe(&stage_changed("a", E3Stage::None, E3Stage::KeyPublished)); + let d = svc.observe(&requested("a")); + assert_eq!( + LifecycleDecision::Regressed { + e3_id: id("a"), + current: E3Stage::KeyPublished, + attempted: E3Stage::Requested + }, + d + ); + // Tracked stage is unchanged. + assert_eq!(E3Stage::KeyPublished, svc.stage(&id("a"))); +} + +#[test] +fn same_stage_is_unchanged() { + let mut svc = E3LifecycleService::new(); + svc.observe(&requested("a")); + let d = svc.observe(&requested("a")); + assert_eq!( + LifecycleDecision::Unchanged { + e3_id: id("a"), + stage: E3Stage::Requested + }, + d + ); +} + +#[test] +fn failure_is_terminal_and_frozen() { + let mut svc = E3LifecycleService::new(); + svc.observe(&requested("a")); + let d = svc.observe(&failed("a", E3Stage::Requested)); + assert_eq!( + LifecycleDecision::Terminal { + e3_id: id("a"), + stage: E3Stage::Failed + }, + d + ); + // Further lifecycle events do not move a terminal E3. + let d2 = svc.observe(&stage_changed("a", E3Stage::Failed, E3Stage::Complete)); + assert_eq!( + LifecycleDecision::Unchanged { + e3_id: id("a"), + stage: E3Stage::Failed + }, + d2 + ); + assert_eq!(E3Stage::Failed, svc.stage(&id("a"))); +} + +#[test] +fn active_excludes_terminal_e3s() { + let mut svc = E3LifecycleService::new(); + svc.observe(&requested("a")); + svc.observe(&requested("b")); + svc.observe(&failed("b", E3Stage::Requested)); + let active = svc.active(); + assert_eq!(vec![id("a")], active); +} + +#[test] +fn non_lifecycle_event_is_ignored() { + let mut svc = E3LifecycleService::new(); + let d = svc.observe(&InterfoldEventData::Shutdown(e3_events::Shutdown)); + assert_eq!(LifecycleDecision::NotLifecycle, d); +} + +#[test] +fn snapshot_roundtrip_preserves_stages() { + let mut svc = E3LifecycleService::new(); + svc.observe(&requested("a")); + svc.observe(&stage_changed( + "a", + E3Stage::Requested, + E3Stage::CiphertextReady, + )); + svc.observe(&requested("b")); + + let snap = svc.snapshot(); + let restored = E3LifecycleService::from_snapshot(snap); + assert_eq!(E3Stage::CiphertextReady, restored.stage(&id("a"))); + assert_eq!(E3Stage::Requested, restored.stage(&id("b"))); +} diff --git a/crates/request/src/routing/actor.rs b/crates/request/src/routing/actor.rs new file mode 100644 index 0000000000..add0339f5e --- /dev/null +++ b/crates/request/src/routing/actor.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::ContextRepositoryFactory; +use crate::E3Context; +use crate::E3ContextParams; +use crate::E3MetaExtension; +use crate::EventBuffer; +use crate::PostForward; +use crate::RequestRouter; +use crate::RouterRepositoryFactory; +use crate::RoutingDecision; +use actix::{Actor, Addr, Context, Handler}; +use anyhow::*; +use async_trait::async_trait; +use e3_data::Checkpoint; +use e3_data::DataStore; +use e3_data::FromSnapshotWithParams; +use e3_data::RepositoriesFactory; +use e3_data::Repository; +use e3_data::Snapshot; +use e3_events::prelude::*; +use e3_events::trap; +use e3_events::BusHandle; +use e3_events::E3RequestComplete; +use e3_events::EType; +use e3_events::EventType; +use e3_events::{E3id, InterfoldEvent}; +use e3_utils::MAILBOX_LIMIT; +use serde::Deserialize; +use serde::Serialize; +use std::collections::HashSet; +use std::{collections::HashMap, sync::Arc}; + +/// An Extension interface for the E3Router system that listens and responds to InterfoldEvents. +/// +/// # Responsibilities +/// - Listens for broadcast InterfoldEvents +/// - Instantiates appropriate actors based on received events +/// - Manages actor state persistence and reconstruction +/// - Handles event streaming to registered recipients +/// +/// # Usage +/// Extensions implement the `on_event` handler to define which events they respond to. +/// When an event is received, the extension typically: +/// 1. Uses the request's context to construct required actors +/// 2. Saves actor addresses to the context using `set_event_recipient` +/// 3. Manages event streaming from buffers to registered recipients +/// +/// Extensions can also reconstruct actors from persisted state using context +/// snapshots and repositories. They can check for dependencies in the context +/// before constructing new extensions. +#[async_trait] +pub trait E3Extension: Send + Sync + 'static { + /// This function is triggered when an InterfoldEvent is sent to the router. Use this to + /// initialize the receiver using `ctx.set_event_receiver(my_address.into())`. Typically this + /// means filtering for specific e3_id enabled events that give rise to actors that have to + /// handle certain behaviour. + fn on_event(&self, ctx: &mut E3Context, evt: &InterfoldEvent); + + /// This function it triggered when the request context is being hydrated from snapshot. + async fn hydrate(&self, ctx: &mut E3Context, snapshot: &crate::E3ContextSnapshot) + -> Result<()>; +} + +/// Routes E3_id-specific contexts to registered extensions and manages message forwarding. +/// +/// # Core Functions +/// - Maintains contexts for each E3 request +/// - Lazily registers extension instances with appropriate dependencies per E3_id +/// - Forwards incoming messages to registered instances +/// - Manages request lifecycle and completion +/// +/// Extensions receive an E3_id-specific context and can handle specific +/// message types. The router ensures proper message delivery and context management +/// throughout the request lifecycle. +/// +/// This actor is a thin message-passing shell: all routing decisions are computed by the +/// pure [`RequestRouter`] service; the actor only performs the resulting actix I/O. +// TODO: setup so that we have to place extensions within correct order of dependencies +pub struct E3Router { + /// The context for every E3 request + contexts: HashMap, + /// A list of completed requests + completed: HashSet, + /// The extensions this instance of the router is configured to listen for + extensions: Arc>>, + /// A buffer for events to send to the + buffer: EventBuffer, + /// The EventBus + bus: BusHandle, + /// A repository for storing snapshots + store: Repository, +} + +pub struct E3RouterParams { + extensions: Arc>>, + bus: BusHandle, + store: Repository, +} + +impl E3Router { + pub fn builder(bus: &BusHandle, store: DataStore) -> E3RouterBuilder { + let repositories = store.repositories(); + let builder = E3RouterBuilder { + bus: bus.clone(), + extensions: vec![], + store: repositories.router(), + }; + + // Everything needs the committe meta factory so adding it here by default + builder.with(E3MetaExtension::create()) + } + + pub fn from_params(params: E3RouterParams) -> Self { + Self { + extensions: params.extensions, + bus: params.bus.clone(), + store: params.store.clone(), + completed: HashSet::new(), + contexts: HashMap::new(), + buffer: EventBuffer::default(), + } + } +} + +#[path = "effects/mod.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +pub use effects::{E3RouterBuilder, E3RouterSnapshot}; + +#[cfg(test)] +#[path = "tests.rs"] +mod tests; diff --git a/crates/request/src/routing/effects/build_context.rs b/crates/request/src/routing/effects/build_context.rs new file mode 100644 index 0000000000..3185c50b78 --- /dev/null +++ b/crates/request/src/routing/effects/build_context.rs @@ -0,0 +1,35 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +use super::*; + +/// Builder for [`E3Router`]. +pub struct E3RouterBuilder { + pub bus: BusHandle, + pub extensions: Vec>, + pub store: Repository, +} + +impl E3RouterBuilder { + pub fn with(mut self, listener: Box) -> Self { + self.extensions.push(listener); + self + } + + pub async fn build(self) -> Result> { + let snapshot: Option = self.store.read().await?; + let params = E3RouterParams { + extensions: self.extensions.into(), + bus: self.bus.clone(), + store: self.store.clone(), + }; + + let router = match snapshot { + Some(snapshot) => E3Router::from_snapshot(params, snapshot).await?, + None => E3Router::from_params(params), + }; + + let addr = router.start(); + self.bus.subscribe(EventType::All, addr.clone().recipient()); + Ok(addr) + } +} diff --git a/crates/request/src/routing/effects/mod.rs b/crates/request/src/routing/effects/mod.rs new file mode 100644 index 0000000000..2ead0c584e --- /dev/null +++ b/crates/request/src/routing/effects/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Router construction and durable snapshot integration. + +use super::*; + +#[path = "build_context.rs"] +mod builder; +mod snapshot; + +pub use builder::E3RouterBuilder; +pub use snapshot::E3RouterSnapshot; diff --git a/crates/request/src/routing/effects/snapshot.rs b/crates/request/src/routing/effects/snapshot.rs new file mode 100644 index 0000000000..44ba9b082c --- /dev/null +++ b/crates/request/src/routing/effects/snapshot.rs @@ -0,0 +1,64 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +use super::*; + +#[derive(Serialize, Deserialize)] +pub struct E3RouterSnapshot { + pub(in crate::actors::router) contexts: Vec, + pub(in crate::actors::router) completed: HashSet, +} + +impl Snapshot for E3Router { + type Snapshot = E3RouterSnapshot; + + fn snapshot(&self) -> Result { + Ok(Self::Snapshot { + contexts: self.contexts.keys().cloned().collect(), + completed: self.completed.clone(), + }) + } +} + +impl Checkpoint for E3Router { + fn repository(&self) -> &Repository { + &self.store + } +} + +#[async_trait] +impl FromSnapshotWithParams for E3Router { + type Params = E3RouterParams; + + async fn from_snapshot(params: Self::Params, snapshot: Self::Snapshot) -> Result { + let mut contexts = HashMap::new(); + let repositories = params.store.repositories(); + + for e3_id in snapshot.contexts { + let Some(context_snapshot) = repositories.context(&e3_id).read().await? else { + continue; + }; + + contexts.insert( + e3_id.clone(), + E3Context::from_snapshot( + E3ContextParams { + repository: repositories.context(&e3_id), + e3_id: e3_id.clone(), + extensions: params.extensions.clone(), + }, + context_snapshot, + ) + .await?, + ); + } + + Ok(E3Router { + contexts, + completed: snapshot.completed, + extensions: params.extensions, + buffer: EventBuffer::default(), + bus: params.bus, + store: params.store, + }) + } +} diff --git a/crates/request/src/domain/event_buffer.rs b/crates/request/src/routing/event_buffer.rs similarity index 100% rename from crates/request/src/domain/event_buffer.rs rename to crates/request/src/routing/event_buffer.rs diff --git a/crates/request/src/routing/handlers.rs b/crates/request/src/routing/handlers.rs new file mode 100644 index 0000000000..b84d18ec82 --- /dev/null +++ b/crates/request/src/routing/handlers.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Mailbox entry points and lifecycle hooks. + +use super::*; + +impl Actor for E3Router { + type Context = Context; + + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT) + } +} + +impl Handler for E3Router { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { + trap( + EType::Event, + &self.bus.with_ec(msg.get_ctx()), + || { + match RequestRouter::route(&msg, &self.completed) { + RoutingDecision::Broadcast => { + for context in self.contexts.values() { + context.forward_message_now(&msg) + } + Ok(()) + } + RoutingDecision::Ignore => Ok(()), + RoutingDecision::AlreadyCompleted(e3_id) => Err(anyhow!( + "unexpected {} for completed E3 {} (event={}, origin={}, source={:?}, block={:?})", + msg.event_type(), + e3_id, + msg.id(), + msg.origin_id(), + msg.source(), + msg.block(), + )), + RoutingDecision::Process { + e3_id, + post_forward, + } => { + let repositories = self.repository().repositories(); + let context = self.contexts.entry(e3_id.clone()).or_insert_with(|| { + E3Context::from_params(E3ContextParams { + e3_id: e3_id.clone(), + repository: repositories.context(&e3_id), + extensions: self.extensions.clone(), + }) + }); + + for extension in self.extensions.iter() { + extension.on_event(context, &msg); + } + + context.forward_message(&msg, &mut self.buffer); + + let (_, ctx) = msg.into_components(); + match post_forward { + PostForward::PublishComplete => { + self.bus.publish( + E3RequestComplete { + e3_id: e3_id.clone(), + }, + ctx, + )?; + } + PostForward::Teardown => { + self.contexts.remove(&e3_id); + self.buffer.remove_e3(&e3_id); + self.completed.insert(e3_id); + } + PostForward::None => (), + } + + self.checkpoint(); + Ok(()) + } + } + }, + ); + } +} diff --git a/crates/request/src/routing/tests.rs b/crates/request/src/routing/tests.rs new file mode 100644 index 0000000000..c1e9b4ec2a --- /dev/null +++ b/crates/request/src/routing/tests.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use crate::E3ContextSnapshot; +use actix::{Actor, Handler}; +use async_trait::async_trait; +use e3_data::{InMemStore, RepositoriesFactory}; +use e3_events::{hlc_factory::HlcFactory, BusHandle, EventBus, Sequencer, StoreEventRequested}; +use std::sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, +}; + +struct StoreSink; + +impl Actor for StoreSink { + type Context = Context; +} + +impl Handler for StoreSink { + type Result = (); + + fn handle(&mut self, _: StoreEventRequested, _: &mut Self::Context) {} +} + +struct RecoveryExtension { + hydrations: Arc, +} + +#[async_trait] +impl E3Extension for RecoveryExtension { + fn on_event(&self, _: &mut E3Context, _: &InterfoldEvent) {} + + async fn hydrate(&self, _: &mut E3Context, _: &E3ContextSnapshot) -> Result<()> { + self.hydrations.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +fn test_bus() -> BusHandle { + let event_bus = EventBus::::default().start(); + let store = StoreSink.start(); + let sequencer = Sequencer::new(&event_bus, store.recipient()).start(); + BusHandle::new(event_bus, sequencer, HlcFactory::new()).enable("router-recovery-test") +} + +#[actix::test] +async fn mid_e3_context_and_completed_set_survive_hydration() -> Result<()> { + let active = E3id::new("7", 31337); + let complete = E3id::new("6", 31337); + let store = DataStore::from_in_mem(&InMemStore::new(false).start()); + let repositories = store.repositories(); + let router_store = repositories.router(); + router_store + .repositories() + .context(&active) + .write_sync(&E3ContextSnapshot { + e3_id: active.clone(), + recipients: vec!["threshold_keyshare".into()], + dependencies: vec!["meta".into()], + }) + .await?; + + let hydrations = Arc::new(AtomicUsize::new(0)); + let params = E3RouterParams { + extensions: Arc::new(vec![Box::new(RecoveryExtension { + hydrations: hydrations.clone(), + })]), + bus: test_bus(), + store: router_store, + }; + let recovered = E3Router::from_snapshot( + params, + E3RouterSnapshot { + contexts: vec![active.clone()], + completed: HashSet::from([complete.clone()]), + }, + ) + .await?; + + assert!(recovered.contexts.contains_key(&active)); + assert!(recovered.completed.contains(&complete)); + assert_eq!(hydrations.load(Ordering::SeqCst), 1); + + let roundtrip = recovered.snapshot()?; + assert_eq!(roundtrip.contexts, vec![active]); + assert_eq!(roundtrip.completed, HashSet::from([complete])); + Ok(()) +} diff --git a/crates/request/src/routing/workflow.rs b/crates/request/src/routing/workflow.rs new file mode 100644 index 0000000000..f001e7688c --- /dev/null +++ b/crates/request/src/routing/workflow.rs @@ -0,0 +1,139 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use e3_events::{E3Stage, E3id, Event, InterfoldEvent, InterfoldEventData}; +use std::collections::HashSet; + +/// The completion action a router should perform *after* running extension hooks and +/// forwarding an event to the request's context. +#[derive(Debug, PartialEq, Eq)] +pub enum PostForward { + /// Publish an `E3RequestComplete` event to the bus: the request has finished. + PublishComplete, + /// Tear down the context for this request and mark it as completed. + Teardown, + /// No completion action is required. + None, +} + +/// The decision the router makes when an event arrives, computed without performing any +/// actix message passing, persistence, or context mutation. +#[derive(Debug, PartialEq, Eq)] +pub enum RoutingDecision { + /// A shutdown event: broadcast it immediately to every active context. + Broadcast, + /// The event carries no `e3_id` and should be ignored. + Ignore, + /// The event targets a request that has already completed; this is an error. + AlreadyCompleted(E3id), + /// Process the event for the given request, applying `post_forward` after forwarding. + Process { + e3_id: E3id, + post_forward: PostForward, + }, +} + +/// Pure routing logic for the E3 request router. +/// +/// Classifies an incoming [`InterfoldEvent`] into a [`RoutingDecision`] based purely on the +/// event data and the set of already-completed requests. This contains no actix, persistence +/// or I/O concerns so it can be unit tested in isolation; the actor executes the decision. +pub struct RequestRouter; + +impl RequestRouter { + /// Decide how an incoming event should be routed given the set of completed requests. + pub fn route(msg: &InterfoldEvent, completed: &HashSet) -> RoutingDecision { + // Broadcast non-E3-scoped lifecycle signals to every active context: + // * `Shutdown` so children can tear themselves down, and + // * `EffectsEnabled` so a hydrated request can re-drive its own in-flight work + // once side effects are switched on at the end of boot sync. + // Both carry no `e3_id`, so without this they would be `Ignore`d and never reach the + // per-E3 child actors. + if matches!( + msg.get_data(), + InterfoldEventData::Shutdown(_) | InterfoldEventData::EffectsEnabled(_) + ) { + return RoutingDecision::Broadcast; + } + + // Durable observational EVM facts are consumed directly by projections + // and global observers. They describe an E3, but they do not drive its + // per-E3 actors. Routing them into a context would create contexts for + // historical observations and report expected post-settlement receipts + // as errors after teardown. + if matches!( + msg.get_data(), + InterfoldEventData::InputPublished(_) + | InterfoldEventData::RewardsDistributed(_) + | InterfoldEventData::RewardCredited(_) + | InterfoldEventData::RewardClaimed(_) + | InterfoldEventData::CommitteeFormationFailed(_) + | InterfoldEventData::CommitteeActivationChanged(_) + | InterfoldEventData::CommitteeViabilityUpdated(_) + | InterfoldEventData::EvmLogObserved(_) + ) { + return RoutingDecision::Ignore; + } + + // Only process events with e3_ids. + let Some(e3_id) = msg.get_e3_id() else { + return RoutingDecision::Ignore; + }; + + // If this e3 round has already been completed then this event is unexpected. + if completed.contains(&e3_id) { + // On-chain confirmation events that lag behind local teardown are expected and + // should be silently ignored rather than treated as an error. + let is_late_terminal = match msg.get_data() { + // E3StageChanged(Complete) always lags local PlaintextAggregated completion. + InterfoldEventData::E3StageChanged(data) + if matches!(data.new_stage, E3Stage::Complete | E3Stage::Failed) => + { + true + } + // E3Failed from on-chain markE3Failed may arrive after a local timeout already + // cleaned up the context. + InterfoldEventData::E3Failed(data) if data.reason.is_timeout() => true, + // Settlement receipts (PlaintextOutputPublished, etc.) can arrive in the same + // EVM block after E3StageChanged(Complete) already tore down the context. + InterfoldEventData::PlaintextOutputPublished(_) => true, + _ => false, + }; + if is_late_terminal { + return RoutingDecision::Ignore; + } + return RoutingDecision::AlreadyCompleted(e3_id); + } + + let post_forward = match msg.get_data() { + // Receiving the PlaintextAggregated event means the request is complete and we can + // notify everyone. This might change as we consider other completion factors. + InterfoldEventData::PlaintextAggregated(_) => PostForward::PublishComplete, + InterfoldEventData::E3StageChanged(data) + if matches!(data.new_stage, E3Stage::Complete) => + { + PostForward::PublishComplete + } + // Timeout failures have no accusation/slashing lifecycle, so the context can be + // torn down immediately. Misbehaviour failures (DKGInvalidShares, etc.) still need + // the accusation/slashing lifecycle to complete before teardown. + InterfoldEventData::E3Failed(data) if data.reason.is_timeout() => { + PostForward::PublishComplete + } + InterfoldEventData::E3RequestComplete(_) => PostForward::Teardown, + _ => PostForward::None, + }; + + RoutingDecision::Process { + e3_id, + post_forward, + } + } +} + +#[cfg(test)] +#[path = "workflow_tests.rs"] +mod tests; diff --git a/crates/request/src/routing/workflow_tests.rs b/crates/request/src/routing/workflow_tests.rs new file mode 100644 index 0000000000..0ca5c2e2e8 --- /dev/null +++ b/crates/request/src/routing/workflow_tests.rs @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use e3_events::{ + E3Failed, E3RequestComplete, E3Stage, E3StageChanged, FailureReason, InterfoldEvent, + PlaintextAggregated, RewardCredited, Sequenced, Shutdown, +}; + +fn e3id() -> E3id { + E3id::new("1", 1) +} + +fn with_e3_id(label: &str, id: E3id) -> InterfoldEvent { + InterfoldEvent::::test_event(label) + .e3_id(id) + .seq(1) + .build() +} + +fn from_data(data: impl Into) -> InterfoldEvent { + InterfoldEvent::::test_event("x") + .data(data) + .seq(1) + .build() +} + +#[test] +fn shutdown_broadcasts() { + let msg = from_data(Shutdown); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Broadcast + ); +} + +#[test] +fn effects_enabled_broadcasts() { + // EffectsEnabled has no e3_id but must reach every hydrated context so each can + // re-drive its own in-flight work after a restart. + let msg = from_data(e3_events::EffectsEnabled::new()); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Broadcast + ); +} + +#[test] +fn event_without_e3_id_is_ignored() { + let msg = InterfoldEvent::::test_event("no-id") + .seq(1) + .build(); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Ignore + ); +} + +#[test] +fn completed_request_is_an_error() { + let id = e3id(); + let mut completed = HashSet::new(); + completed.insert(id.clone()); + let msg = with_e3_id("late", id.clone()); + assert_eq!( + RequestRouter::route(&msg, &completed), + RoutingDecision::AlreadyCompleted(id) + ); +} + +#[test] +fn settlement_receipt_is_not_routed_to_completed_context() { + let id = e3id(); + let mut completed = HashSet::new(); + completed.insert(id.clone()); + let msg = from_data(RewardCredited { + e3_id: id, + account: "0x01".into(), + token: "0x02".into(), + amount: "10".into(), + }); + + assert_eq!( + RequestRouter::route(&msg, &completed), + RoutingDecision::Ignore + ); +} + +#[test] +fn stage_changed_to_complete_ignored_when_already_completed() { + // E3StageChanged(Complete) arriving from the EVM after local teardown is expected — + // the on-chain confirmation lags behind local completion. It should be silently + // ignored, not treated as an error. + let id = e3id(); + let mut completed = HashSet::new(); + completed.insert(id.clone()); + let msg = from_data(E3StageChanged { + e3_id: id.clone(), + previous_stage: E3Stage::CiphertextReady, + new_stage: E3Stage::Complete, + }); + assert_eq!( + RequestRouter::route(&msg, &completed), + RoutingDecision::Ignore + ); +} + +#[test] +fn stage_changed_to_failed_ignored_when_completed() { + // E3StageChanged(Failed) from the EVM can arrive after a local timeout already cleaned up + // the context. Treat it as a silent no-op, the same way we handle E3StageChanged(Complete). + let id = e3id(); + let mut completed = HashSet::new(); + completed.insert(id.clone()); + let msg = from_data(E3StageChanged { + e3_id: id.clone(), + previous_stage: E3Stage::CiphertextReady, + new_stage: E3Stage::Failed, + }); + assert_eq!( + RequestRouter::route(&msg, &completed), + RoutingDecision::Ignore + ); +} + +#[test] +fn plaintext_aggregated_publishes_complete() { + let id = e3id(); + let msg = from_data(PlaintextAggregated { + e3_id: id.clone(), + decrypted_output: vec![], + decryption_aggregator_proofs: vec![], + }); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Process { + e3_id: id, + post_forward: PostForward::PublishComplete, + } + ); +} + +#[test] +fn stage_changed_to_complete_publishes_complete() { + let id = e3id(); + let msg = from_data(E3StageChanged { + e3_id: id.clone(), + previous_stage: E3Stage::CiphertextReady, + new_stage: E3Stage::Complete, + }); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Process { + e3_id: id, + post_forward: PostForward::PublishComplete, + } + ); +} + +#[test] +fn stage_changed_to_failed_does_not_complete() { + let id = e3id(); + let msg = from_data(E3StageChanged { + e3_id: id.clone(), + previous_stage: E3Stage::CiphertextReady, + new_stage: E3Stage::Failed, + }); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Process { + e3_id: id, + post_forward: PostForward::None, + } + ); +} + +#[test] +fn e3_request_complete_triggers_teardown() { + // InterfoldEventData::get_e3_id() now returns Some(e3_id) for E3RequestComplete, + // so the event reaches the Teardown arm of the router. + let id = e3id(); + let msg = from_data(E3RequestComplete { e3_id: id.clone() }); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Process { + e3_id: id, + post_forward: PostForward::Teardown, + } + ); +} + +#[test] +fn generic_event_with_e3_id_has_no_completion() { + let id = e3id(); + let msg = with_e3_id("generic", id.clone()); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Process { + e3_id: id, + post_forward: PostForward::None, + } + ); +} + +// --- timeout-triggered E3Failed tests --- + +fn e3_failed(id: E3id, reason: FailureReason) -> InterfoldEvent { + from_data(E3Failed { + e3_id: id, + failed_at_stage: E3Stage::CommitteeFinalized, + reason, + }) +} + +#[test] +fn e3_failed_dkg_timeout_publishes_complete() { + let id = e3id(); + let msg = e3_failed(id.clone(), FailureReason::DKGTimeout); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Process { + e3_id: id, + post_forward: PostForward::PublishComplete, + } + ); +} + +#[test] +fn e3_failed_committee_formation_timeout_publishes_complete() { + let id = e3id(); + let msg = e3_failed(id.clone(), FailureReason::CommitteeFormationTimeout); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Process { + e3_id: id, + post_forward: PostForward::PublishComplete, + } + ); +} + +#[test] +fn e3_failed_compute_timeout_publishes_complete() { + let id = e3id(); + let msg = e3_failed(id.clone(), FailureReason::ComputeTimeout); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Process { + e3_id: id, + post_forward: PostForward::PublishComplete, + } + ); +} + +#[test] +fn e3_failed_decryption_timeout_publishes_complete() { + let id = e3id(); + let msg = e3_failed(id.clone(), FailureReason::DecryptionTimeout); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Process { + e3_id: id, + post_forward: PostForward::PublishComplete, + } + ); +} + +#[test] +fn e3_failed_invalid_shares_does_not_complete() { + // Slashable failures must NOT trigger E3RequestComplete — the accusation/slashing + // lifecycle must be allowed to finish first. + let id = e3id(); + let msg = e3_failed(id.clone(), FailureReason::DKGInvalidShares); + assert_eq!( + RequestRouter::route(&msg, &HashSet::new()), + RoutingDecision::Process { + e3_id: id, + post_forward: PostForward::None, + } + ); +} + +#[test] +fn e3_failed_timeout_ignored_when_already_completed() { + let id = e3id(); + let mut completed = HashSet::new(); + completed.insert(id.clone()); + let msg = e3_failed(id.clone(), FailureReason::DKGTimeout); + assert_eq!( + RequestRouter::route(&msg, &completed), + RoutingDecision::Ignore + ); +} + +#[test] +fn stage_changed_to_failed_ignored_when_already_completed() { + let id = e3id(); + let mut completed = HashSet::new(); + completed.insert(id.clone()); + let msg = from_data(E3StageChanged { + e3_id: id.clone(), + previous_stage: E3Stage::CommitteeFinalized, + new_stage: E3Stage::Failed, + }); + assert_eq!( + RequestRouter::route(&msg, &completed), + RoutingDecision::Ignore + ); +} diff --git a/crates/slashing/src/accusation_voting/actor.rs b/crates/slashing/src/accusation_voting/actor.rs new file mode 100644 index 0000000000..b782d639d0 --- /dev/null +++ b/crates/slashing/src/accusation_voting/actor.rs @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Off-chain accusation quorum protocol for fault attribution. +//! +//! When a node detects a ZK proof failure from another committee member, it +//! broadcasts a [`ProofFailureAccusation`] over gossip. Other committee members +//! independently check the same proof and respond with [`AccusationVote`]s. +//! Once the on-chain quorum `H` votes is reached, the +//! actor emits [`AccusationQuorumReached`] for downstream consumers (aggregator +//! exclusion, on-chain slash submission). +//! +//! ## Architecture +//! +//! This file is a **thin actix shell**. All protocol logic lives in the plain, +//! synchronous [`AccusationVoting`] service ([`crate::accusation_voting`]). The +//! actor's only job is to translate inbound [`InterfoldEvent`]s into service +//! calls and to perform the I/O ([`VoteAction`]s) the service returns — +//! publishing gossip events, dispatching ZK requests, and managing vote +//! timeouts. +//! +//! ## Proof-type-specific behavior +//! +//! | Proof | Attestation | Notes | +//! |---------|----------------------------|--------------------------------------------| +//! | C0 | All nodes independently | Everyone receives via DHT | +//! | C1 | All nodes independently | Bundled in ThresholdShareCreated | +//! | C2a/C2b | All nodes independently | Same proof bytes for all recipients | +//! | C3a/C3b | Forwarding required | Per-recipient; accuser forwards payload | +//! | C4 | All nodes independently | Broadcast via gossip | +//! | C5 | Committee attests | Aggregator-generated; nodes verify off-chain| +//! | C6 | All nodes independently | Broadcast via gossip | +//! | C7 | On-chain verification | Not handled here (on-chain verifier) | + +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; + +use actix::{Actor, Addr, AsyncContext, Context, Handler, SpawnHandle}; +use alloy::primitives::{Address, Bytes}; +use alloy::signers::local::PrivateKeySigner; +use e3_events::{ + AccusationVote, BusHandle, CommitmentConsistencyViolation, ComputeRequestError, + ComputeResponse, E3id, EventPublisher, EventSubscriber, EventType, InterfoldEvent, + InterfoldEventData, ProofFailureAccusation, ProofType, ProofVerificationFailed, + ProofVerificationPassed, TypedEvent, +}; +use e3_utils::NotifySync; +use e3_zk_helpers::CiphernodesCommitteeSize; +use tracing::{error, warn}; + +pub use crate::workflow::accusation_voting::Clock; +use crate::workflow::accusation_voting::{AccusationVoting, VoteAction}; + +/// Production clock backed by `SystemTime::now()`. +pub struct SystemClock; + +impl Clock for SystemClock { + fn unix_now_secs(&self) -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) + } +} + +/// Thin actix shell around the [`AccusationVoting`] domain service. +/// +/// **Lifecycle**: One instance per E3 computation. Created by +/// [`AccusationManagerExtension`] when [`CommitteeFinalized`] fires and +/// destroyed when the E3 completes or the node shuts down. All protocol state +/// lives inside the owned [`AccusationVoting`] service and is therefore +/// naturally scoped to a single E3. +/// +/// **Ephemeral**: This actor does *not* persist state across restarts. +/// In-flight accusations are lost on node restart (accepted trade-off: +/// they would have timed out within the vote timeout anyway). +/// +/// Subscribes to: +/// - [`ProofVerificationFailed`] — local proof failure detection +/// - [`ProofVerificationPassed`] — cache successful verification for voting +/// - [`ProofFailureAccusation`] — incoming accusations from other nodes via gossip +/// - [`AccusationVote`] — incoming votes from other nodes via gossip +/// - [`SlashExecuted`] — on-chain slash confirmation for committee updates +/// +/// Publishes: +/// - [`ProofFailureAccusation`] — broadcast own accusations via gossip +/// - [`AccusationVote`] — broadcast own votes via gossip +/// - [`AccusationQuorumReached`] — quorum decision for downstream consumers +/// +/// [`AccusationManagerExtension`]: crate::accusation_manager_ext::AccusationManagerExtension +/// [`CommitteeFinalized`]: e3_events::CommitteeFinalized +/// [`AccusationQuorumReached`]: e3_events::AccusationQuorumReached +/// [`SlashExecuted`]: e3_events::SlashExecuted +pub struct AccusationManager { + bus: BusHandle, + /// Plain, synchronous protocol core. Owns all accusation/vote state. + voting: AccusationVoting, + /// Active vote-collection timeouts keyed by accusation_id. Managed entirely + /// by the actor — the service only signals start/cancel via [`VoteAction`]. + timeout_handles: HashMap<[u8; 32], SpawnHandle>, +} + +#[path = "effects/mod.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +#[cfg(test)] +#[path = "actor_tests.rs"] +mod tests; diff --git a/crates/slashing/src/accusation_voting/actor_tests.rs b/crates/slashing/src/accusation_voting/actor_tests.rs new file mode 100644 index 0000000000..d48e9f329f --- /dev/null +++ b/crates/slashing/src/accusation_voting/actor_tests.rs @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use alloy::primitives::{keccak256, FixedBytes, U256}; +use alloy::signers::SignerSync; +use alloy::sol_types::SolValue; +use e3_events::{VOTE_DOMAIN_NAME, VOTE_DOMAIN_VERSION, VOTE_TYPEHASH_STR}; +use e3_utils::ArcBytes; + +/// Default clock-skew allowance when validating peer-stamped accusation +/// deadlines (mirrors the production extension default). +const DEFAULT_ACCUSATION_DEADLINE_SKEW_SECS: u64 = 30; + +/// Independent re-derivation of the EIP-712 vote digest, mirroring exactly +/// what `SlashingManager._verifyVotes` computes on chain. +fn reference_vote_digest( + chain_id: u64, + verifying_contract: Address, + e3_id: u64, + accusation_id: [u8; 32], + voter: Address, + data_hash: [u8; 32], + deadline: u64, +) -> [u8; 32] { + let domain_typehash: [u8; 32] = keccak256( + "EIP712Domain(string name,string version,uint256 chainId,address verifyingContract)", + ) + .into(); + let name_hash: [u8; 32] = keccak256(VOTE_DOMAIN_NAME).into(); + let version_hash: [u8; 32] = keccak256(VOTE_DOMAIN_VERSION).into(); + let domain_separator: [u8; 32] = keccak256( + ( + domain_typehash, + name_hash, + version_hash, + U256::from(chain_id), + verifying_contract, + ) + .abi_encode(), + ) + .into(); + + let typehash: [u8; 32] = keccak256(VOTE_TYPEHASH_STR).into(); + let struct_hash: [u8; 32] = keccak256( + ( + typehash, + U256::from(e3_id), + FixedBytes::<32>::from(accusation_id), + voter, + FixedBytes::<32>::from(data_hash), + U256::from(deadline), + ) + .abi_encode(), + ) + .into(); + + let mut buf = Vec::with_capacity(2 + 32 + 32); + buf.push(0x19); + buf.push(0x01); + buf.extend_from_slice(&domain_separator); + buf.extend_from_slice(&struct_hash); + keccak256(&buf).into() +} + +/// The actor's `vote_digest` must equal the reference digest byte-for-byte. +#[test] +fn vote_digest_matches_reference() { + let chain_id = 31337u64; + let verifying_contract: Address = "0x9999999999999999999999999999999999999999" + .parse() + .unwrap(); + let voter: Address = "0x2222222222222222222222222222222222222222" + .parse() + .unwrap(); + let accusation_id = [0xab; 32]; + let data_hash = [0xcd; 32]; + let deadline: u64 = 1_700_000_000; + + let vote = AccusationVote { + e3_id: E3id::new("42", chain_id), + accusation_id, + voter, + data_hash, + deadline, + signature: ArcBytes::default(), + }; + + let actor = AccusationManager::vote_digest(&vote, verifying_contract); + let reference = reference_vote_digest( + chain_id, + verifying_contract, + 42, + accusation_id, + voter, + data_hash, + deadline, + ); + + assert_eq!( + actor, reference, + "AccusationManager::vote_digest drifted from the reference EIP-712 \ + computation. Check VOTE_TYPEHASH_STR / VOTE_DOMAIN_NAME against \ + SlashingManager.sol — these MUST stay byte-equal across crates." + ); +} + +/// Sign-and-recover round-trip using the actor's digest. +#[test] +fn actor_signature_recovers_to_voter() { + let signer: PrivateKeySigner = + "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + .parse() + .unwrap(); + let voter = signer.address(); + let verifying_contract: Address = "0x5555555555555555555555555555555555555555" + .parse() + .unwrap(); + let chain_id = 31337u64; + + let vote = AccusationVote { + e3_id: E3id::new("12345", chain_id), + accusation_id: [0x07; 32], + voter, + data_hash: [0x08; 32], + deadline: 1_700_000_000, + signature: ArcBytes::default(), + }; + + let digest = AccusationManager::vote_digest(&vote, verifying_contract); + let sig = signer + .sign_hash_sync(&FixedBytes::<32>::from(digest)) + .unwrap(); + let recovered = sig + .recover_address_from_prehash(&FixedBytes::<32>::from(digest)) + .expect("recover"); + assert_eq!( + recovered, voter, + "signing the actor's digest and recovering must yield the voter" + ); +} + +/// The accusation digest must include `deadline`. +#[test] +fn accusation_digest_binds_deadline() { + let make = |deadline: u64| ProofFailureAccusation { + e3_id: E3id::new("9", 31337), + accuser: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + .parse() + .unwrap(), + accused: "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + .parse() + .unwrap(), + accused_party_id: 1, + proof_type: ProofType::C1PkGeneration, + data_hash: [0x42; 32], + deadline, + signed_payload: None, + signature: ArcBytes::default(), + }; + let a = AccusationVoting::accusation_digest(&make(1_700_000_000)); + let b = AccusationVoting::accusation_digest(&make(1_700_000_001)); + assert_ne!(a, b, "deadline must be part of the accusation digest"); +} + +#[test] +fn peer_deadline_acceptance_enforces_local_window() { + let now = 1_700_000_000u64; + let validity = 1_800u64; + let skew = DEFAULT_ACCUSATION_DEADLINE_SKEW_SECS; + let max_ok = now + validity + skew; + + assert!( + !AccusationVoting::is_peer_deadline_acceptable(now, now, validity, skew), + "deadline equal to now must be rejected" + ); + assert!( + !AccusationVoting::is_peer_deadline_acceptable(now - 1, now, validity, skew), + "expired deadline must be rejected" + ); + assert!( + AccusationVoting::is_peer_deadline_acceptable(max_ok, now, validity, skew), + "deadline at upper bound must be accepted" + ); + assert!( + !AccusationVoting::is_peer_deadline_acceptable(max_ok + 1, now, validity, skew), + "far-future deadline must be rejected" + ); + assert!( + !AccusationVoting::is_peer_deadline_acceptable(now + 10, now, 0, skew), + "vote_validity_secs=0 must reject peer accusations" + ); +} diff --git a/crates/slashing/src/accusation_voting/effects/apply_actions.rs b/crates/slashing/src/accusation_voting/effects/apply_actions.rs new file mode 100644 index 0000000000..1cf552f7fc --- /dev/null +++ b/crates/slashing/src/accusation_voting/effects/apply_actions.rs @@ -0,0 +1,73 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Execute workflow effects through the event bus and owned Actix timers. + +use super::*; + +impl AccusationManager { + /// Perform the I/O the [`AccusationVoting`] service requested. + /// + /// This is the *only* place the actor publishes events or touches timers — + /// keeping all protocol decisions in the pure service. + pub(in crate::actors::accusation_manager) fn apply_actions( + &mut self, + actions: Vec, + ctx: &mut Context, + ) { + for action in actions { + match action { + VoteAction::PublishAccusation { + accusation, + ec, + dedup_key, + } => { + if let Err(err) = self.bus.publish(accusation, ec) { + error!("Failed to broadcast ProofFailureAccusation: {err}"); + // Preserve the original rollback: re-allow this + // (accused, proof_type) accusation on a dead bus. + self.voting.rollback_initiation(&dedup_key); + } + } + VoteAction::PublishVote { vote, ec } => { + if let Err(err) = self.bus.publish(vote, ec) { + error!("Failed to broadcast AccusationVote: {err}"); + } + } + VoteAction::PublishQuorum { quorum, ec } => { + if let Err(err) = self.bus.publish(quorum, ec) { + error!("Failed to publish AccusationQuorumReached: {err}"); + } + } + VoteAction::DispatchZk { + request, + ec, + correlation_id, + } => { + if let Err(err) = self.bus.publish(request, ec) { + error!("Failed to dispatch C3a/C3b ZK re-verification: {err}"); + self.voting.discard_reverification(&correlation_id); + } + } + VoteAction::StartTimeout(accusation_id) => { + let timeout = self.voting.vote_timeout(); + let handle = ctx.run_later(timeout, move |act, _ctx| { + act.timeout_handles.remove(&accusation_id); + if let Some((quorum, ec)) = act.voting.on_vote_timeout(accusation_id) { + if let Err(err) = act.bus.publish(quorum, ec) { + error!( + "Failed to publish AccusationQuorumReached on timeout: {err}" + ); + } + } + }); + self.timeout_handles.insert(accusation_id, handle); + } + VoteAction::CancelTimeout(accusation_id) => { + if let Some(handle) = self.timeout_handles.remove(&accusation_id) { + ctx.cancel_future(handle); + } + } + } + } + } +} diff --git a/crates/slashing/src/accusation_voting/effects/construction.rs b/crates/slashing/src/accusation_voting/effects/construction.rs new file mode 100644 index 0000000000..5278b9b596 --- /dev/null +++ b/crates/slashing/src/accusation_voting/effects/construction.rs @@ -0,0 +1,213 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Construction, subscription, digest compatibility, and verifier-cache entry points. + +use super::*; + +impl AccusationManager { + pub(in crate::actors::accusation_manager) fn canonical_vote_quorum( + circuit_threshold_t: usize, + committee_n: usize, + ) -> usize { + match CiphernodesCommitteeSize::from_threshold(circuit_threshold_t, committee_n) { + Ok(size) => size.values().h, + Err(err) => { + // Preserve the historical constructor behavior for external callers with a + // non-canonical test committee. Production creation is validated by the + // extension and uses `setup_with_quorum` below. + warn!( + circuit_threshold_t, + committee_n, + error = %err, + "Unknown committee size; falling back to the supplied threshold as vote quorum" + ); + circuit_threshold_t + } + } + } + + /// Construct an actor with the production [`SystemClock`]. Use + /// [`AccusationManager::new_with_clock`] in tests that need deterministic + /// timestamps. + #[allow(clippy::too_many_arguments)] + pub fn new( + bus: &BusHandle, + e3_id: E3id, + signer: PrivateKeySigner, + slashing_manager: Address, + committee: Vec

, + circuit_threshold_t: usize, + vote_validity_secs: u64, + accusation_deadline_skew_secs: u64, + params_preset: e3_fhe_params::BfvPreset, + ) -> Self { + Self::new_with_clock( + bus, + e3_id, + signer, + slashing_manager, + committee, + circuit_threshold_t, + vote_validity_secs, + accusation_deadline_skew_secs, + params_preset, + Arc::new(SystemClock), + ) + } + + /// Construct an actor with an explicit [`Clock`]. Allows unit tests to + /// drive deadline computation without touching wall-clock time. + #[allow(clippy::too_many_arguments)] + pub fn new_with_clock( + bus: &BusHandle, + e3_id: E3id, + signer: PrivateKeySigner, + slashing_manager: Address, + committee: Vec
, + circuit_threshold_t: usize, + vote_validity_secs: u64, + accusation_deadline_skew_secs: u64, + params_preset: e3_fhe_params::BfvPreset, + clock: Arc, + ) -> Self { + let vote_quorum_h = Self::canonical_vote_quorum(circuit_threshold_t, committee.len()); + Self::new_with_clock_and_quorum( + bus, + e3_id, + signer, + slashing_manager, + committee, + circuit_threshold_t, + vote_quorum_h, + vote_validity_secs, + accusation_deadline_skew_secs, + params_preset, + clock, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(in crate::actors::accusation_manager) fn new_with_clock_and_quorum( + bus: &BusHandle, + e3_id: E3id, + signer: PrivateKeySigner, + slashing_manager: Address, + committee: Vec
, + circuit_threshold_t: usize, + vote_quorum_h: usize, + vote_validity_secs: u64, + accusation_deadline_skew_secs: u64, + params_preset: e3_fhe_params::BfvPreset, + clock: Arc, + ) -> Self { + Self { + bus: bus.clone(), + voting: AccusationVoting::new( + e3_id, + signer, + slashing_manager, + committee, + circuit_threshold_t, + vote_quorum_h, + vote_validity_secs, + accusation_deadline_skew_secs, + params_preset, + clock, + ), + timeout_handles: HashMap::new(), + } + } + + #[allow(clippy::too_many_arguments)] + pub fn setup( + bus: &BusHandle, + e3_id: E3id, + signer: PrivateKeySigner, + slashing_manager: Address, + committee: Vec
, + circuit_threshold_t: usize, + vote_validity_secs: u64, + accusation_deadline_skew_secs: u64, + params_preset: e3_fhe_params::BfvPreset, + ) -> Addr { + let vote_quorum_h = Self::canonical_vote_quorum(circuit_threshold_t, committee.len()); + Self::setup_with_quorum( + bus, + e3_id, + signer, + slashing_manager, + committee, + circuit_threshold_t, + vote_quorum_h, + vote_validity_secs, + accusation_deadline_skew_secs, + params_preset, + ) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn setup_with_quorum( + bus: &BusHandle, + e3_id: E3id, + signer: PrivateKeySigner, + slashing_manager: Address, + committee: Vec
, + circuit_threshold_t: usize, + vote_quorum_h: usize, + vote_validity_secs: u64, + accusation_deadline_skew_secs: u64, + params_preset: e3_fhe_params::BfvPreset, + ) -> Addr { + let addr = Self::new_with_clock_and_quorum( + bus, + e3_id, + signer, + slashing_manager, + committee, + circuit_threshold_t, + vote_quorum_h, + vote_validity_secs, + accusation_deadline_skew_secs, + params_preset, + Arc::new(SystemClock), + ) + .start(); + bus.subscribe(EventType::ProofVerificationFailed, addr.clone().into()); + bus.subscribe(EventType::ProofVerificationPassed, addr.clone().into()); + bus.subscribe(EventType::ProofFailureAccusation, addr.clone().into()); + bus.subscribe(EventType::AccusationVote, addr.clone().into()); + bus.subscribe(EventType::ComputeResponse, addr.clone().into()); + bus.subscribe(EventType::ComputeRequestError, addr.clone().into()); + bus.subscribe(EventType::SlashExecuted, addr.clone().into()); + bus.subscribe( + EventType::CommitmentConsistencyViolation, + addr.clone().into(), + ); + addr + } + + /// Canonical EIP-712 typed-data hash for a vote. + /// + /// Delegates to [`AccusationVoting::vote_digest`]. Exposed `pub` so the + /// Anvil parity test in + /// `crates/zk-prover/tests/slashing_integration_tests.rs` can sign votes + /// through the **same** code path the production actor uses. + pub fn vote_digest(vote: &AccusationVote, verifying_contract: Address) -> [u8; 32] { + AccusationVoting::vote_digest(vote, verifying_contract) + } + + /// Cache a successful proof verification result for a specific + /// (accused, proof_type). Allows the node to vote on accusations from + /// other nodes. + pub fn cache_verification_result( + &mut self, + accused: Address, + proof_type: ProofType, + data_hash: [u8; 32], + passed: bool, + evidence: Bytes, + ) { + self.voting + .cache_verification_result(accused, proof_type, data_hash, passed, evidence); + } +} diff --git a/crates/sortition/src/actors/mod.rs b/crates/slashing/src/accusation_voting/effects/mod.rs similarity index 66% rename from crates/sortition/src/actors/mod.rs rename to crates/slashing/src/accusation_voting/effects/mod.rs index 6701e7e348..45700e67f4 100644 --- a/crates/sortition/src/actors/mod.rs +++ b/crates/slashing/src/accusation_voting/effects/mod.rs @@ -4,8 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -mod ciphernode_selector; -mod sortition; +//! Effect execution for accusation voting. -pub use ciphernode_selector::*; -pub use sortition::*; +use super::*; + +mod apply_actions; +mod construction; diff --git a/crates/slashing/src/accusation_voting/handlers.rs b/crates/slashing/src/accusation_voting/handlers.rs new file mode 100644 index 0000000000..83d03cd2c5 --- /dev/null +++ b/crates/slashing/src/accusation_voting/handlers.rs @@ -0,0 +1,134 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Actix envelope routing into the accusation workflow. + +use super::*; + +impl Actor for AccusationManager { + type Context = Context; +} + +impl Handler for AccusationManager { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + match msg { + InterfoldEventData::ProofVerificationFailed(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ProofVerificationPassed(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ProofFailureAccusation(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::AccusationVote(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ComputeResponse(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ComputeRequestError(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::SlashExecuted(data) => { + self.voting.on_slash_executed(data); + } + InterfoldEventData::CommitmentConsistencyViolation(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + _ => (), + } + } +} + +impl Handler> for AccusationManager { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let (data, ec) = msg.into_components(); + let actions = self.voting.on_local_proof_failure(data, &ec); + self.apply_actions(actions, ctx); + } +} + +impl Handler> for AccusationManager { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (data, _ec) = msg.into_components(); + self.voting.on_proof_verification_passed(data); + } +} + +impl Handler> for AccusationManager { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let (data, ec) = msg.into_components(); + let actions = self.voting.on_accusation_received(data, &ec); + self.apply_actions(actions, ctx); + } +} + +impl Handler> for AccusationManager { + type Result = (); + + fn handle(&mut self, msg: TypedEvent, ctx: &mut Self::Context) -> Self::Result { + let (data, ec) = msg.into_components(); + let actions = self.voting.on_vote_received(data, &ec); + self.apply_actions(actions, ctx); + } +} + +impl Handler> for AccusationManager { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let actions = self.voting.handle_reverification_response(msg); + self.apply_actions(actions, ctx); + } +} + +impl Handler> for AccusationManager { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.voting.handle_reverification_error(msg); + } +} + +impl Handler> for AccusationManager { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + let (data, ec) = msg.into_components(); + let actions = self.voting.on_consistency_violation(data, &ec); + self.apply_actions(actions, ctx); + } +} diff --git a/crates/slashing/src/accusation_voting/state.rs b/crates/slashing/src/accusation_voting/state.rs new file mode 100644 index 0000000000..7abf1d996f --- /dev/null +++ b/crates/slashing/src/accusation_voting/state.rs @@ -0,0 +1,82 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Accusation workflow state, inputs, and effect intents. + +use super::*; + +/// How long to wait for votes before declaring the accusation inconclusive. +pub(crate) const DEFAULT_VOTE_TIMEOUT: Duration = Duration::from_secs(300); + +/// Injected time source used by deterministic deadline transitions. +pub trait Clock: Send + Sync + 'static { + fn unix_now_secs(&self) -> u64; +} + +/// An I/O effect the actor must perform on behalf of [`AccusationVoting`]. +pub(crate) enum VoteAction { + PublishAccusation { + accusation: ProofFailureAccusation, + ec: EventContext, + dedup_key: (Address, ProofType), + }, + PublishVote { + vote: AccusationVote, + ec: EventContext, + }, + PublishQuorum { + quorum: AccusationQuorumReached, + ec: EventContext, + }, + DispatchZk { + request: ComputeRequest, + ec: EventContext, + correlation_id: CorrelationId, + }, + StartTimeout([u8; 32]), + CancelTimeout([u8; 32]), +} + +/// An active accusation awaiting agreement votes from committee members. +pub(crate) struct PendingAccusation { + pub(crate) accusation: ProofFailureAccusation, + pub(crate) votes_for: Vec, + pub(crate) ec: EventContext, +} + +/// Cached verification result for an accused party and proof type. +pub(super) struct ReceivedProofData { + pub(super) data_hash: [u8; 32], + pub(super) verification_passed: bool, + pub(super) evidence: Bytes, +} + +/// An in-flight ZK re-verification for a forwarded C3a/C3b proof. +pub(super) struct PendingReVerification { + pub(super) accusation_id: [u8; 32], + pub(super) data_hash: [u8; 32], + pub(super) accused: Address, + pub(super) proof_type: ProofType, + pub(super) evidence: Bytes, +} + +/// Pure, synchronous core of the accusation quorum protocol. +pub(crate) struct AccusationVoting { + pub(super) e3_id: E3id, + pub(super) my_address: Address, + pub(super) signer: PrivateKeySigner, + pub(super) slashing_manager: Address, + pub(super) committee: Vec
, + pub(super) circuit_threshold_t: usize, + pub(super) vote_quorum_h: usize, + pub(super) committee_n: usize, + pub(super) pending: HashMap<[u8; 32], PendingAccusation>, + pub(super) accused_proofs: HashSet<(Address, ProofType)>, + pub(super) received_data: HashMap<(Address, ProofType), ReceivedProofData>, + pub(super) buffered_votes: HashMap<[u8; 32], Vec>, + pub(super) pending_reverifications: HashMap, + pub(super) vote_timeout: Duration, + pub(super) vote_validity_secs: u64, + pub(super) accusation_deadline_skew_secs: u64, + pub(super) clock: Arc, + pub(super) params_preset: e3_fhe_params::BfvPreset, +} diff --git a/crates/slashing/src/accusation_voting/transitions/incoming_accusations.rs b/crates/slashing/src/accusation_voting/transitions/incoming_accusations.rs new file mode 100644 index 0000000000..37bbb47b84 --- /dev/null +++ b/crates/slashing/src/accusation_voting/transitions/incoming_accusations.rs @@ -0,0 +1,283 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Incoming accusation validation and local vote decisions. + +use super::*; + +impl AccusationVoting { + /// Called when we receive an accusation from another node via gossip. + pub(crate) fn on_accusation_received( + &mut self, + accusation: ProofFailureAccusation, + ec: &EventContext, + ) -> Vec { + let mut actions = Vec::new(); + self.on_accusation_received_inner(accusation, ec, &mut actions); + actions + } + + pub(super) fn on_accusation_received_inner( + &mut self, + accusation: ProofFailureAccusation, + ec: &EventContext, + actions: &mut Vec, + ) { + // Ignore accusations for other E3s + if accusation.e3_id != self.e3_id { + return; + } + + let now = self.clock.unix_now_secs(); + if !Self::is_peer_deadline_acceptable( + accusation.deadline, + now, + self.vote_validity_secs, + self.accusation_deadline_skew_secs, + ) { + let max_deadline = now + .saturating_add(self.vote_validity_secs) + .saturating_add(self.accusation_deadline_skew_secs); + warn!( + "Ignoring accusation from {} — deadline {} outside local validity window \ + (now={}, vote_validity_secs={}, skew_secs={}, max_accepted_deadline={})", + accusation.accuser, + accusation.deadline, + now, + self.vote_validity_secs, + self.accusation_deadline_skew_secs, + max_deadline + ); + return; + } + + // Verify accuser is in committee + if !self.committee.contains(&accusation.accuser) { + warn!( + "Ignoring accusation from non-committee member {}", + accusation.accuser + ); + return; + } + + // Verify accused is a committee member (defense-in-depth) + if !self.committee.contains(&accusation.accused) { + warn!( + "Ignoring accusation against non-committee member {}", + accusation.accused + ); + return; + } + + // Ignore our own accusations (we already voted) + if accusation.accuser == self.my_address { + return; + } + + // Verify accuser's ECDSA signature + if !self.verify_accusation_signature(&accusation) { + warn!( + "Invalid signature on accusation from {} — ignoring", + accusation.accuser + ); + return; + } + + let accusation_id = Self::accusation_id(&accusation); + + // Don't process duplicate accusations + if self.pending.contains_key(&accusation_id) { + return; + } + + // Determine our position based on our local verification state. + let key = (accusation.accused, accusation.proof_type); + let our_data_hash = if let Some(received) = self.received_data.get(&key) { + if received.verification_passed { + info!( + "Local verification of {:?} from {} passed — abstaining \ + (no disagreement vote on the wire)", + accusation.proof_type, accusation.accused + ); + return; + } + received.data_hash + } else if let Some(ref forwarded) = accusation.signed_payload { + // C3a/C3b case: we didn't receive this proof directly. + let forwarded_valid = match forwarded.recover_address() { + Ok(addr) => { + if addr != accusation.accused { + warn!( + "Forwarded C3a/C3b payload signer {} != accused {} — cannot verify", + addr, accusation.accused + ); + false + } else if forwarded.payload.e3_id != self.e3_id { + warn!("Forwarded C3a/C3b payload e3_id mismatch — cannot verify"); + false + } else { + let expected = forwarded.payload.proof_type.circuit_names(); + expected.contains(&forwarded.payload.proof.circuit) + } + } + Err(e) => { + warn!("Forwarded C3a/C3b payload signature invalid: {e} — cannot verify"); + false + } + }; + + if !forwarded_valid { + // Can't trust the forwarded proof — abstain + return; + } + + // Bind the forwarded proof to the accusation. + if forwarded.payload.proof_type != accusation.proof_type { + warn!( + "Forwarded C3a/C3b proof_type {:?} != accusation proof_type {:?} — cannot verify", + forwarded.payload.proof_type, accusation.proof_type + ); + return; + } + let computed_hash = Self::compute_payload_hash(forwarded); + if computed_hash != accusation.data_hash { + warn!( + "Forwarded C3a/C3b data_hash mismatch (len {} vs {}) — cannot verify", + computed_hash.len(), + accusation.data_hash.len() + ); + return; + } + + let data_hash = Self::compute_payload_hash(forwarded); + let evidence: Bytes = ( + Bytes::copy_from_slice(&forwarded.payload.proof.data), + Bytes::copy_from_slice(&forwarded.payload.proof.public_signals), + ) + .abi_encode() + .into(); + let accused_party_id = accusation.accused_party_id; + let forwarded_clone = forwarded.clone(); + + let committee_size = match CiphernodesCommitteeSize::from_threshold( + self.circuit_threshold_t, + self.committee_n, + ) { + Ok(c) => c, + Err(e) => { + warn!("Cannot derive committee size for ZK re-verification: {e}"); + return; + } + }; + + // Create PendingAccusation without our vote — it arrives after ZK completes. + actions.push(VoteAction::StartTimeout(accusation_id)); + self.pending.insert( + accusation_id, + PendingAccusation { + accusation, + votes_for: Vec::new(), + ec: ec.clone(), + }, + ); + + // Replay any buffered votes + if let Some(buffered) = self.buffered_votes.remove(&accusation_id) { + for vote in buffered { + self.on_vote_received_inner(vote, ec, actions); + } + } + + // Dispatch ZK re-verification + let correlation_id = CorrelationId::new(); + self.pending_reverifications.insert( + correlation_id, + PendingReVerification { + accusation_id, + data_hash, + accused: key.0, + proof_type: key.1, + evidence, + }, + ); + + let party_proof = PartyProofsToVerify { + sender_party_id: accused_party_id, + signed_proofs: vec![forwarded_clone], + }; + let request = ComputeRequest::zk( + ZkRequest::VerifyShareProofs(VerifyShareProofsRequest { + party_proofs: vec![party_proof], + params_preset: self.params_preset, + committee_size, + }), + correlation_id, + self.e3_id.clone(), + ); + + actions.push(VoteAction::DispatchZk { + request, + ec: ec.clone(), + correlation_id, + }); + + // Vote deferred — return without falling through to the normal vote path + return; + } else { + // We don't have the data and no payload was forwarded — abstain + info!( + "No local data for accused {} proof {:?} — abstaining from vote", + accusation.accused, accusation.proof_type + ); + return; + }; + + // We saw the proof fail locally — agree with the accusation. + let mut vote = AccusationVote { + e3_id: self.e3_id.clone(), + accusation_id, + voter: self.my_address, + data_hash: our_data_hash, + deadline: accusation.deadline, + signature: ArcBytes::default(), + }; + match self.sign_vote_digest(&vote) { + Ok(sig) => vote.signature = ArcBytes::from_bytes(&sig), + Err(err) => { + error!("Failed to sign AccusationVote: {err}"); + return; + } + } + + info!( + "Agreeing with accusation against {} for {:?}", + accusation.accused, accusation.proof_type + ); + + // Broadcast vote via gossip + actions.push(VoteAction::PublishVote { + vote: vote.clone(), + ec: ec.clone(), + }); + + // Start timeout for this accusation + actions.push(VoteAction::StartTimeout(accusation_id)); + + // Record in pending + let pending = PendingAccusation { + accusation, + votes_for: vec![vote], + ec: ec.clone(), + }; + self.pending.insert(accusation_id, pending); + + // Replay any votes that arrived before this accusation + if let Some(buffered) = self.buffered_votes.remove(&accusation_id) { + for vote in buffered { + self.on_vote_received_inner(vote, ec, actions); + } + } + + // Check quorum + self.check_quorum(accusation_id, ec, actions); + } +} diff --git a/crates/slashing/src/accusation_voting/transitions/initiate_accusation.rs b/crates/slashing/src/accusation_voting/transitions/initiate_accusation.rs new file mode 100644 index 0000000000..ba3a5381d4 --- /dev/null +++ b/crates/slashing/src/accusation_voting/transitions/initiate_accusation.rs @@ -0,0 +1,248 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Local fault/consistency inputs and accusation creation. + +use super::*; + +impl AccusationVoting { + /// Called when the local node detects a proof failure. + pub(crate) fn on_local_proof_failure( + &mut self, + event: ProofVerificationFailed, + ec: &EventContext, + ) -> Vec { + if event.e3_id != self.e3_id { + return Vec::new(); + } + + let accused_address = if event.accused_address == Address::ZERO { + if let Some(&addr) = self.committee.get(event.accused_party_id as usize) { + warn!( + "Resolved Address::ZERO for party {} to committee address {}", + event.accused_party_id, addr + ); + addr + } else { + error!( + "Cannot resolve address for party {} (out of committee bounds) — dropping accusation", + event.accused_party_id + ); + return Vec::new(); + } + } else { + event.accused_address + }; + + if !self.committee.contains(&accused_address) { + warn!( + "Ignoring proof failure for {} — not on E3 {} committee", + accused_address, self.e3_id + ); + return Vec::new(); + } + + // Cache the failed verification result. + let evidence = Bytes::from( + ( + Bytes::copy_from_slice(&event.signed_payload.payload.proof.data), + Bytes::copy_from_slice(&event.signed_payload.payload.proof.public_signals), + ) + .abi_encode(), + ); + self.received_data.insert( + (accused_address, event.proof_type), + ReceivedProofData { + data_hash: event.data_hash, + verification_passed: false, + evidence, + }, + ); + + // For C3a/C3b, include the signed payload so other nodes can re-verify + let forwarded_payload = match event.proof_type { + ProofType::C3aSkShareEncryption | ProofType::C3bESmShareEncryption => { + Some(event.signed_payload.clone()) + } + _ => None, + }; + + let mut actions = Vec::new(); + self.initiate_accusation( + accused_address, + event.accused_party_id, + event.proof_type, + event.data_hash, + forwarded_payload, + ec, + &mut actions, + ); + actions + } + + /// Called when the `CommitmentConsistencyChecker` detects a cross-circuit + /// commitment mismatch for a party. + pub(crate) fn on_consistency_violation( + &mut self, + data: CommitmentConsistencyViolation, + ec: &EventContext, + ) -> Vec { + if data.e3_id != self.e3_id { + return Vec::new(); + } + + if !self.committee.contains(&data.accused_address) { + warn!( + "Ignoring commitment violation for {} — not on E3 {} committee", + data.accused_address, self.e3_id + ); + return Vec::new(); + } + + self.received_data.insert( + (data.accused_address, data.proof_type), + ReceivedProofData { + data_hash: data.data_hash, + verification_passed: false, + evidence: data.evidence.clone(), + }, + ); + + let mut actions = Vec::new(); + self.initiate_accusation( + data.accused_address, + data.accused_party_id, + data.proof_type, + data.data_hash, + None, + ec, + &mut actions, + ); + actions + } + + /// Shared accusation creation and broadcast logic. + #[allow(clippy::too_many_arguments)] + pub(super) fn initiate_accusation( + &mut self, + accused_address: Address, + accused_party_id: u64, + proof_type: ProofType, + data_hash: [u8; 32], + forwarded_payload: Option, + ec: &EventContext, + actions: &mut Vec, + ) { + if !self.committee.contains(&accused_address) { + warn!( + "Refusing accusation against {} — not on E3 {} committee", + accused_address, self.e3_id + ); + return; + } + + let key = (accused_address, proof_type); + + // Dedup: don't create multiple accusations for the same (accused, proof_type) + if !self.accused_proofs.insert(key) { + info!( + "Already accused {:?} for {:?} — skipping duplicate", + accused_address, proof_type + ); + return; + } + + // Governance-disabled validity window means no accusation voting. + if self.vote_validity_secs == 0 { + warn!( + "Refusing accusation initiation for {:?} on E3 {}: vote_validity_secs is 0", + accused_address, self.e3_id + ); + self.accused_proofs.remove(&key); + return; + } + + // Pick the on-chain validity deadline once per accusation. + let deadline = self.compute_deadline(); + + // Create the accusation + let mut accusation = ProofFailureAccusation { + e3_id: self.e3_id.clone(), + accuser: self.my_address, + accused: accused_address, + accused_party_id, + proof_type, + data_hash, + deadline, + signed_payload: forwarded_payload, + signature: ArcBytes::default(), + }; + match self.sign_accusation_digest(&accusation) { + Ok(sig) => accusation.signature = ArcBytes::from_bytes(&sig), + Err(err) => { + error!("Failed to sign ProofFailureAccusation: {err}"); + self.accused_proofs.remove(&key); + return; + } + } + + let accusation_id = Self::accusation_id(&accusation); + + info!( + "Broadcasting accusation against {} for {:?} failure", + accused_address, proof_type + ); + + // Broadcast accusation via gossip + actions.push(VoteAction::PublishAccusation { + accusation: accusation.clone(), + ec: ec.clone(), + dedup_key: key, + }); + + // Cast our own agreement vote (we just observed the failure locally). + let mut own_vote = AccusationVote { + e3_id: self.e3_id.clone(), + accusation_id, + voter: self.my_address, + data_hash, + deadline, + signature: ArcBytes::default(), + }; + match self.sign_vote_digest(&own_vote) { + Ok(sig) => own_vote.signature = ArcBytes::from_bytes(&sig), + Err(err) => { + error!("Failed to sign own AccusationVote: {err}"); + self.accused_proofs.remove(&key); + return; + } + } + + actions.push(VoteAction::PublishVote { + vote: own_vote.clone(), + ec: ec.clone(), + }); + + // Start timeout + actions.push(VoteAction::StartTimeout(accusation_id)); + + // Store pending accusation with own vote + self.pending.insert( + accusation_id, + PendingAccusation { + accusation, + votes_for: vec![own_vote], + ec: ec.clone(), + }, + ); + + // Replay any votes that arrived before this accusation + if let Some(buffered) = self.buffered_votes.remove(&accusation_id) { + for vote in buffered { + self.on_vote_received_inner(vote, ec, actions); + } + } + + // Check quorum immediately (defensive for a future H=1 committee). + self.check_quorum(accusation_id, ec, actions); + } +} diff --git a/crates/slashing/src/accusation_voting/transitions/mod.rs b/crates/slashing/src/accusation_voting/transitions/mod.rs new file mode 100644 index 0000000000..c8831a9f6c --- /dev/null +++ b/crates/slashing/src/accusation_voting/transitions/mod.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Deterministic accusation transitions grouped by protocol input. + +use super::*; + +#[path = "incoming_accusations.rs"] +mod incoming; +#[path = "initiate_accusation.rs"] +mod initiation; +#[path = "reverify_proofs.rs"] +mod reverification; +mod setup; +#[path = "vote.rs"] +mod voting; diff --git a/crates/slashing/src/accusation_voting/transitions/reverify_proofs.rs b/crates/slashing/src/accusation_voting/transitions/reverify_proofs.rs new file mode 100644 index 0000000000..ab14973bdf --- /dev/null +++ b/crates/slashing/src/accusation_voting/transitions/reverify_proofs.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Slash cleanup and forwarded-proof re-verification results. + +use super::*; + +impl AccusationVoting { + /// Handle an on-chain SlashExecuted event for this E3. + pub(crate) fn on_slash_executed(&mut self, data: SlashExecuted) { + if data.e3_id != self.e3_id { + return; + } + let prev_len = self.committee.len(); + self.committee.retain(|addr| *addr != data.operator); + if self.committee.len() < prev_len { + info!( + "Removed slashed operator {} from committee (now {} members)", + data.operator, + self.committee.len() + ); + + // Purge any votes from the expelled node in pending accusations + for pending in self.pending.values_mut() { + pending.votes_for.retain(|v| v.voter != data.operator); + } + + // Purge from buffered votes + for buf in self.buffered_votes.values_mut() { + buf.retain(|v| v.voter != data.operator); + } + } + } + + /// Handle ZK re-verification response for a forwarded C3a/C3b proof. + pub(crate) fn handle_reverification_response( + &mut self, + msg: TypedEvent, + ) -> Vec { + let (msg, _ec) = msg.into_components(); + let mut actions = Vec::new(); + + let correlation_id = msg.correlation_id; + let Some(reverif) = self.pending_reverifications.remove(&correlation_id) else { + return actions; // Not our correlation ID + }; + + let zk_passed = match msg.response { + ComputeResponseKind::Zk(ZkResponse::VerifyShareProofs(r)) => { + if r.party_results.is_empty() { + warn!("Empty ZK re-verification results — abstaining"); + return actions; + } + r.party_results.first().is_some_and(|r| r.all_verified) + } + _ => { + warn!("Unexpected ComputeResponse kind for C3a/C3b re-verification — abstaining"); + return actions; + } + }; + + // Cache the result for future accusations regardless of outcome. + self.cache_verification_result( + reverif.accused, + reverif.proof_type, + reverif.data_hash, + zk_passed, + reverif.evidence.clone(), + ); + + // ZK re-verification passed ⇒ proof is valid ⇒ we disagree ⇒ abstain. + if zk_passed { + info!( + "C3a/C3b re-verification passed for {:?} — abstaining from vote", + reverif.proof_type + ); + return actions; + } + + // ZK re-verification failed ⇒ we agree with the accusation. + let (ec, deadline) = match self.pending.get(&reverif.accusation_id) { + Some(pending) => (pending.ec.clone(), pending.accusation.deadline), + None => { + // Accusation already resolved before ZK finished + return actions; + } + }; + + let mut vote = AccusationVote { + e3_id: self.e3_id.clone(), + accusation_id: reverif.accusation_id, + voter: self.my_address, + data_hash: reverif.data_hash, + deadline, + signature: ArcBytes::default(), + }; + match self.sign_vote_digest(&vote) { + Ok(sig) => vote.signature = ArcBytes::from_bytes(&sig), + Err(err) => { + error!("Failed to sign C3a/C3b AccusationVote: {err}"); + return actions; + } + } + + info!( + "C3a/C3b re-verification confirmed failure for {:?} — agreeing with accusation", + reverif.proof_type + ); + + // Broadcast vote via gossip + actions.push(VoteAction::PublishVote { + vote: vote.clone(), + ec: ec.clone(), + }); + + // Record in pending + if let Some(pending) = self.pending.get_mut(&reverif.accusation_id) { + pending.votes_for.push(vote); + } + + // Check quorum + self.check_quorum(reverif.accusation_id, &ec, &mut actions); + actions + } + + /// Handle ZK re-verification error for a forwarded C3a/C3b proof. + pub(crate) fn handle_reverification_error(&mut self, msg: TypedEvent) { + let (msg, _ec) = msg.into_components(); + + let correlation_id = msg.correlation_id(); + let Some(reverif) = self.pending_reverifications.remove(correlation_id) else { + return; // Not our correlation ID + }; + + error!( + "C3a/C3b ZK re-verification failed for {:?} — abstaining from vote", + reverif.proof_type + ); + // Don't vote — effectively abstain + } +} diff --git a/crates/slashing/src/accusation_voting/transitions/setup.rs b/crates/slashing/src/accusation_voting/transitions/setup.rs new file mode 100644 index 0000000000..2716e30494 --- /dev/null +++ b/crates/slashing/src/accusation_voting/transitions/setup.rs @@ -0,0 +1,292 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Workflow construction, EIP-712 digests, deadlines, and verification cache. + +use super::*; + +impl AccusationVoting { + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + e3_id: E3id, + signer: PrivateKeySigner, + slashing_manager: Address, + committee: Vec
, + circuit_threshold_t: usize, + vote_quorum_h: usize, + vote_validity_secs: u64, + accusation_deadline_skew_secs: u64, + params_preset: e3_fhe_params::BfvPreset, + clock: Arc, + ) -> Self { + let my_address = signer.address(); + let committee_n = committee.len(); + Self { + e3_id, + my_address, + signer, + slashing_manager, + committee, + circuit_threshold_t, + vote_quorum_h, + committee_n, + pending: HashMap::new(), + accused_proofs: HashSet::new(), + received_data: HashMap::new(), + buffered_votes: HashMap::new(), + pending_reverifications: HashMap::new(), + vote_timeout: DEFAULT_VOTE_TIMEOUT, + vote_validity_secs, + accusation_deadline_skew_secs, + clock, + params_preset, + } + } + + /// The vote-collection timeout the actor should schedule. + pub(crate) fn vote_timeout(&self) -> Duration { + self.vote_timeout + } + + // ─── Deadline computation ──────────────────────────────────────────── + + /// Compute the on-chain vote-validity deadline (Unix seconds) the accuser + /// stamps on a fresh accusation. + pub(super) fn compute_deadline(&self) -> u64 { + self.clock + .unix_now_secs() + .saturating_add(self.vote_validity_secs) + } + + /// Validate a peer-provided accusation deadline against this node's local + /// vote-validity policy and wall clock. + pub(crate) fn is_peer_deadline_acceptable( + deadline: u64, + now: u64, + vote_validity_secs: u64, + skew_secs: u64, + ) -> bool { + if vote_validity_secs == 0 { + return false; + } + let max_deadline = now + .saturating_add(vote_validity_secs) + .saturating_add(skew_secs); + deadline > now && deadline <= max_deadline + } + + // ─── Accusation ID computation ─────────────────────────────────────── + + /// Compute a deterministic ID for an accusation based on its key fields. + /// + /// `keccak256(abi.encodePacked(chainId, e3Id, accused, proofType))` + pub(crate) fn accusation_id(accusation: &ProofFailureAccusation) -> [u8; 32] { + let e3_id_u256: U256 = accusation + .e3_id + .clone() + .try_into() + .expect("E3id should be valid U256"); + let msg = ( + U256::from(accusation.e3_id.chain_id()), + e3_id_u256, + accusation.accused, + U256::from(accusation.proof_type as u8), + ) + .abi_encode_packed(); + keccak256(&msg).into() + } + + // ─── Signing / Verification ────────────────────────────────────────── + + pub(super) fn sign_accusation_digest( + &self, + accusation: &ProofFailureAccusation, + ) -> Result, alloy::signers::Error> { + let digest = Self::accusation_digest(accusation); + let sig = self.signer.sign_message_sync(&digest)?; + Ok(sig.as_bytes().to_vec()) + } + + /// Structured digest for ECDSA signing of accusations. Off-chain only. + pub(crate) fn accusation_digest(accusation: &ProofFailureAccusation) -> [u8; 32] { + let e3_id_u256: U256 = accusation + .e3_id + .clone() + .try_into() + .expect("E3id should be valid U256"); + let typehash: [u8; 32] = keccak256( + "ProofFailureAccusation(uint256 chainId,uint256 e3Id,address accuser,address accused,uint256 proofType,bytes32 dataHash,uint256 deadline)" + ).into(); + let encoded = ( + typehash, + U256::from(accusation.e3_id.chain_id()), + e3_id_u256, + accusation.accuser, + accusation.accused, + U256::from(accusation.proof_type as u8), + accusation.data_hash, + U256::from(accusation.deadline), + ) + .abi_encode(); + keccak256(&encoded).into() + } + + pub(super) fn verify_accusation_signature(&self, accusation: &ProofFailureAccusation) -> bool { + let digest = Self::accusation_digest(accusation); + let sig = match alloy::primitives::Signature::try_from( + accusation.signature.extract_bytes().as_ref(), + ) { + Ok(s) => s, + Err(_) => return false, + }; + match sig.recover_address_from_msg(digest) { + Ok(addr) => addr == accusation.accuser, + Err(_) => false, + } + } + + pub(super) fn sign_vote_digest( + &self, + vote: &AccusationVote, + ) -> Result, alloy::signers::Error> { + let digest = Self::vote_digest(vote, self.slashing_manager); + // `sign_hash_sync` signs the raw 32-byte hash without EIP-191 wrapping, + // which is what EIP-712 requires. + let sig = self.signer.sign_hash_sync(&digest.into())?; + Ok(sig.as_bytes().to_vec()) + } + + /// Canonical EIP-712 domain separator for vote signatures. + /// + /// Must match `SlashingManager`'s domain construction exactly. + pub(super) fn vote_domain_separator(chain_id: u64, verifying_contract: Address) -> [u8; 32] { + let domain_typehash: [u8; 32] = keccak256( + "EIP712Domain(string name,string version,uint256 chainId,address verifyingContract)", + ) + .into(); + let name_hash: [u8; 32] = keccak256(VOTE_DOMAIN_NAME).into(); + let version_hash: [u8; 32] = keccak256(VOTE_DOMAIN_VERSION).into(); + let encoded = ( + domain_typehash, + name_hash, + version_hash, + U256::from(chain_id), + verifying_contract, + ) + .abi_encode(); + keccak256(&encoded).into() + } + + /// Canonical EIP-712 typed-data hash for a vote. + /// + /// `keccak256("\x19\x01" || domainSeparator || structHash)`. + pub(crate) fn vote_digest(vote: &AccusationVote, verifying_contract: Address) -> [u8; 32] { + let e3_id_u256: U256 = vote + .e3_id + .clone() + .try_into() + .expect("E3id should be valid U256"); + let typehash: [u8; 32] = keccak256(VOTE_TYPEHASH_STR).into(); + let struct_hash: [u8; 32] = keccak256( + ( + typehash, + e3_id_u256, + vote.accusation_id, + vote.voter, + vote.data_hash, + U256::from(vote.deadline), + ) + .abi_encode(), + ) + .into(); + let domain = Self::vote_domain_separator(vote.e3_id.chain_id(), verifying_contract); + let mut buf = Vec::with_capacity(2 + 32 + 32); + buf.push(0x19); + buf.push(0x01); + buf.extend_from_slice(&domain); + buf.extend_from_slice(&struct_hash); + keccak256(&buf).into() + } + + pub(super) fn verify_vote_signature(&self, vote: &AccusationVote) -> bool { + let digest = Self::vote_digest(vote, self.slashing_manager); + let sig = + match alloy::primitives::Signature::try_from(vote.signature.extract_bytes().as_ref()) { + Ok(s) => s, + Err(_) => return false, + }; + match sig.recover_address_from_prehash(&digest.into()) { + Ok(addr) => addr == vote.voter, + Err(_) => false, + } + } + + /// Compute a keccak256 hash of a SignedProofPayload for data_hash comparison. + pub(super) fn compute_payload_hash(payload: &SignedProofPayload) -> [u8; 32] { + let msg = ( + Bytes::copy_from_slice(&payload.payload.proof.data), + Bytes::copy_from_slice(&payload.payload.proof.public_signals), + ) + .abi_encode(); + keccak256(&msg).into() + } + + // ─── Caching ───────────────────────────────────────────────────────── + + /// Cache a successful (or failed) proof verification result. + pub(crate) fn cache_verification_result( + &mut self, + accused: Address, + proof_type: ProofType, + data_hash: [u8; 32], + passed: bool, + evidence: Bytes, + ) { + self.received_data.insert( + (accused, proof_type), + ReceivedProofData { + data_hash, + verification_passed: passed, + evidence, + }, + ); + } + + /// Cache a successful proof verification reported via `ProofVerificationPassed`. + pub(crate) fn on_proof_verification_passed(&mut self, data: ProofVerificationPassed) { + if data.e3_id != self.e3_id { + return; + } + if !self.committee.contains(&data.address) { + return; + } + // Evidence preimage = `abi.encode(proof.data, public_signals)`. + let evidence: Bytes = ( + Bytes::copy_from_slice(&data.proof_data), + Bytes::copy_from_slice(&data.public_signals), + ) + .abi_encode() + .into(); + self.received_data.insert( + (data.address, data.proof_type), + ReceivedProofData { + data_hash: data.data_hash, + verification_passed: true, + evidence, + }, + ); + } + + // ─── Rollback helpers (publish-failure paths) ──────────────────────── + + /// Roll back an initiation whose accusation broadcast failed. Mirrors the + /// original actor's behaviour of removing the dedup entry so a future + /// identical failure may retry. + pub(crate) fn rollback_initiation(&mut self, dedup_key: &(Address, ProofType)) { + self.accused_proofs.remove(dedup_key); + } + + /// Discard a pending ZK re-verification whose dispatch failed. + pub(crate) fn discard_reverification(&mut self, correlation_id: &CorrelationId) { + self.pending_reverifications.remove(correlation_id); + } +} diff --git a/crates/slashing/src/accusation_voting/transitions/vote.rs b/crates/slashing/src/accusation_voting/transitions/vote.rs new file mode 100644 index 0000000000..3e6e833fce --- /dev/null +++ b/crates/slashing/src/accusation_voting/transitions/vote.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Vote admission, quorum decisions, timeouts, and terminal actions. + +use super::*; + +impl AccusationVoting { + /// Called when we receive a vote from another node via gossip. + pub(crate) fn on_vote_received( + &mut self, + vote: AccusationVote, + ec: &EventContext, + ) -> Vec { + let mut actions = Vec::new(); + self.on_vote_received_inner(vote, ec, &mut actions); + actions + } + + pub(super) fn on_vote_received_inner( + &mut self, + vote: AccusationVote, + ec: &EventContext, + actions: &mut Vec, + ) { + // Ignore votes for other E3s + if vote.e3_id != self.e3_id { + return; + } + + // Verify voter is in committee + if !self.committee.contains(&vote.voter) { + warn!("Ignoring vote from non-committee member {}", vote.voter); + return; + } + + // Ignore our own votes (already recorded) + if vote.voter == self.my_address { + return; + } + + // Verify voter's ECDSA signature + if !self.verify_vote_signature(&vote) { + warn!("Invalid signature on vote from {} — ignoring", vote.voter); + return; + } + + let vote_accusation_id = vote.accusation_id; + + // Find the pending accusation + let Some(pending) = self.pending.get_mut(&vote_accusation_id) else { + // Unknown accusation — buffer the vote for replay. + let committee_len = self.committee.len(); + let buf = self.buffered_votes.entry(vote_accusation_id).or_default(); + if buf.len() < committee_len { + buf.push(vote); + } else { + warn!( + "Buffered votes for unknown accusation {:?} reached committee-size cap — dropping vote", + vote_accusation_id + ); + } + return; + }; + + // Reject votes whose deadline disagrees with the accusation's deadline. + if vote.deadline != pending.accusation.deadline { + warn!( + "Ignoring vote from {} — deadline {} does not match accusation deadline {}", + vote.voter, vote.deadline, pending.accusation.deadline + ); + return; + } + + // Reject votes from the accused party — conflict of interest + if vote.voter == pending.accusation.accused { + warn!( + "Ignoring vote from accused party {} on their own accusation", + vote.voter + ); + return; + } + + // Dedup: don't count same voter twice + let already_voted = pending.votes_for.iter().any(|v| v.voter == vote.voter); + if already_voted { + return; + } + + // Accuser's vote data_hash must match the accusation's data_hash. + if vote.voter == pending.accusation.accuser + && vote.data_hash != pending.accusation.data_hash + { + warn!( + "Accuser {} sent vote with data_hash inconsistent with their accusation — rejecting vote", + vote.voter + ); + return; + } + + // Every received `AccusationVote` is an agreement. + pending.votes_for.push(vote); + + self.check_quorum(vote_accusation_id, ec, actions); + } + + /// Evaluate whether we have enough agreeing votes to decide. + pub(super) fn check_quorum( + &mut self, + accusation_id: [u8; 32], + ec: &EventContext, + actions: &mut Vec, + ) { + let Some(pending) = self.pending.get(&accusation_id) else { + return; + }; + + let agree_count = pending.votes_for.len(); + if agree_count < self.vote_quorum_h { + // Not yet at quorum. + return; + } + + // Reached `H` — decide between AccusedFaulted and Equivocation. + let agree_hashes: HashSet<[u8; 32]> = + pending.votes_for.iter().map(|v| v.data_hash).collect(); + if agree_hashes.len() > 1 { + info!( + "Equivocation detected at quorum: {} unique data hashes among {} agreeing voters for {} {:?}", + agree_hashes.len(), + agree_count, + pending.accusation.accused, + pending.accusation.proof_type + ); + self.emit_quorum_reached(accusation_id, AccusationOutcome::Equivocation, ec, actions); + } else { + info!( + "Quorum reached: {} votes confirm {} sent bad {:?} proof — AccusedFaulted", + agree_count, pending.accusation.accused, pending.accusation.proof_type + ); + self.emit_quorum_reached( + accusation_id, + AccusationOutcome::AccusedFaulted, + ec, + actions, + ); + } + } + + /// Called when the vote timeout expires for an accusation. Returns the + /// terminal quorum event the actor must publish, if the accusation was + /// still pending. + pub(crate) fn on_vote_timeout( + &mut self, + accusation_id: [u8; 32], + ) -> Option<(AccusationQuorumReached, EventContext)> { + let pending = self.pending.remove(&accusation_id)?; // Already resolved + + let outcome = if pending.votes_for.len() >= self.vote_quorum_h { + let agree_hashes: HashSet<[u8; 32]> = + pending.votes_for.iter().map(|v| v.data_hash).collect(); + if agree_hashes.len() > 1 { + AccusationOutcome::Equivocation + } else { + AccusationOutcome::AccusedFaulted + } + } else { + AccusationOutcome::Inconclusive + }; + + warn!( + "Accusation against {} for {:?} timed out with {} agreeing votes — outcome: {:?}", + pending.accusation.accused, + pending.accusation.proof_type, + pending.votes_for.len(), + outcome + ); + + let evidence = self + .received_data + .get(&(pending.accusation.accused, pending.accusation.proof_type)) + .map(|d| d.evidence.clone()) + .unwrap_or_default(); + Some(( + AccusationQuorumReached { + e3_id: self.e3_id.clone(), + accuser: pending.accusation.accuser, + accused: pending.accusation.accused, + proof_type: pending.accusation.proof_type, + votes_for: pending.votes_for, + outcome, + evidence, + }, + pending.ec, + )) + } + + pub(super) fn emit_quorum_reached( + &mut self, + accusation_id: [u8; 32], + outcome: AccusationOutcome, + ec: &EventContext, + actions: &mut Vec, + ) { + let Some(pending) = self.pending.remove(&accusation_id) else { + return; + }; + + // Cancel the timeout to avoid unnecessary timer fires + actions.push(VoteAction::CancelTimeout(accusation_id)); + + info!( + "Accusation quorum reached for {} {:?}: {} agreeing votes — outcome: {}", + pending.accusation.accused, + pending.accusation.proof_type, + pending.votes_for.len(), + outcome + ); + + let evidence = self + .received_data + .get(&(pending.accusation.accused, pending.accusation.proof_type)) + .map(|d| d.evidence.clone()) + .unwrap_or_default(); + actions.push(VoteAction::PublishQuorum { + quorum: AccusationQuorumReached { + e3_id: self.e3_id.clone(), + accuser: pending.accusation.accuser, + accused: pending.accusation.accused, + proof_type: pending.accusation.proof_type, + votes_for: pending.votes_for, + outcome, + evidence, + }, + ec: ec.clone(), + }); + } +} diff --git a/crates/slashing/src/accusation_voting/workflow.rs b/crates/slashing/src/accusation_voting/workflow.rs new file mode 100644 index 0000000000..baf83c6d28 --- /dev/null +++ b/crates/slashing/src/accusation_voting/workflow.rs @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Deterministic workflow for the off-chain accusation quorum +//! protocol. +//! +//! This module contains **all** the business logic that used to live inside +//! the `AccusationManager` actix actor: +//! +//! - EIP-712 digest computation (accusation + vote) +//! - ECDSA signature creation and verification +//! - deadline stamping / peer-deadline validation +//! - pending-accusation bookkeeping (votes, dedup, buffering) +//! - vote tallying, quorum threshold checks, and equivocation detection +//! +//! [`AccusationVoting`] owns the protocol state and exposes plain methods that +//! mutate that state and **return a list of [`VoteAction`]s** describing the +//! I/O the actor must perform (publish a gossip event, dispatch a ZK request, +//! start/cancel a vote timeout). The service itself performs **no** I/O: it +//! never touches the event bus, the actix context, or timers. This makes the +//! whole protocol deterministically unit-testable without spinning up an actor +//! system. + +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; +use std::time::Duration; + +use alloy::primitives::{keccak256, Address, Bytes, U256}; +use alloy::signers::local::PrivateKeySigner; +use alloy::signers::SignerSync; +use alloy::sol_types::SolValue; +use e3_events::{ + AccusationOutcome, AccusationQuorumReached, AccusationVote, CommitmentConsistencyViolation, + ComputeRequest, ComputeRequestError, ComputeResponse, ComputeResponseKind, CorrelationId, E3id, + EventContext, PartyProofsToVerify, ProofFailureAccusation, ProofType, ProofVerificationFailed, + ProofVerificationPassed, Sequenced, SignedProofPayload, SlashExecuted, TypedEvent, + VerifyShareProofsRequest, ZkRequest, ZkResponse, VOTE_DOMAIN_NAME, VOTE_DOMAIN_VERSION, + VOTE_TYPEHASH_STR, +}; +use e3_utils::ArcBytes; +use e3_zk_helpers::CiphernodesCommitteeSize; +use tracing::{error, info, warn}; + +#[path = "state.rs"] +mod state; +#[path = "transitions/mod.rs"] +mod transitions; + +pub use state::Clock; +pub(crate) use state::*; + +// ════════════════════════════════════════════════════════════════════════════ +// Tests +// ════════════════════════════════════════════════════════════════════════════ + +#[cfg(test)] +#[path = "workflow_tests/mod.rs"] +mod tests; diff --git a/crates/slashing/src/accusation_voting/workflow_tests/mod.rs b/crates/slashing/src/accusation_voting/workflow_tests/mod.rs new file mode 100644 index 0000000000..4a1da3d8ee --- /dev/null +++ b/crates/slashing/src/accusation_voting/workflow_tests/mod.rs @@ -0,0 +1,123 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use alloy::primitives::FixedBytes; +use e3_events::{InterfoldEventData, Unsequenced}; + +struct FixedClock(u64); +impl Clock for FixedClock { + fn unix_now_secs(&self) -> u64 { + self.0 + } +} + +/// A throwaway sequenced [`EventContext`] for driving service calls in +/// tests. The service never inspects the context beyond cloning it onto +/// emitted actions, so any well-formed origin context works. +fn ctx() -> EventContext { + let vote = AccusationVote { + e3_id: E3id::new("42", CHAIN_ID), + accusation_id: [0u8; 32], + voter: Address::ZERO, + data_hash: [0u8; 32], + deadline: 0, + signature: ArcBytes::default(), + }; + EventContext::::from(InterfoldEventData::from(vote)).sequence(0) +} + +const CHAIN_ID: u64 = 31337; +const VALIDITY: u64 = 1_800; +const SKEW: u64 = 30; +const NOW: u64 = 1_700_000_000; + +fn signer(byte: u8) -> PrivateKeySigner { + let mut bytes = [0u8; 32]; + bytes[31] = byte; + PrivateKeySigner::from_bytes(&FixedBytes::<32>::from(bytes)).unwrap() +} + +fn voting_with( + me: &PrivateKeySigner, + committee: Vec
, + circuit_threshold_t: usize, + vote_quorum_h: usize, +) -> AccusationVoting { + AccusationVoting::new( + E3id::new("42", CHAIN_ID), + me.clone(), + "0x9999999999999999999999999999999999999999" + .parse() + .unwrap(), + committee, + circuit_threshold_t, + vote_quorum_h, + VALIDITY, + SKEW, + e3_fhe_params::BfvPreset::default(), + Arc::new(FixedClock(NOW)), + ) +} + +/// Build and sign a vote as `who` for the given accusation/data hash. +fn signed_vote( + who: &PrivateKeySigner, + slashing_manager: Address, + e3_id: &E3id, + accusation_id: [u8; 32], + data_hash: [u8; 32], + deadline: u64, +) -> AccusationVote { + let mut vote = AccusationVote { + e3_id: e3_id.clone(), + accusation_id, + voter: who.address(), + data_hash, + deadline, + signature: ArcBytes::default(), + }; + let digest = AccusationVoting::vote_digest(&vote, slashing_manager); + let sig = who.sign_hash_sync(&FixedBytes::<32>::from(digest)).unwrap(); + vote.signature = ArcBytes::from_bytes(&sig.as_bytes()); + vote +} + +fn insert_pending( + v: &mut AccusationVoting, + accuser: &PrivateKeySigner, + accused: Address, + data_hash: [u8; 32], + deadline: u64, + own_vote: AccusationVote, +) -> [u8; 32] { + let accusation = ProofFailureAccusation { + e3_id: v.e3_id.clone(), + accuser: accuser.address(), + accused, + accused_party_id: 1, + proof_type: ProofType::C1PkGeneration, + data_hash, + deadline, + signed_payload: None, + signature: ArcBytes::default(), + }; + let id = AccusationVoting::accusation_id(&accusation); + v.pending.insert( + id, + PendingAccusation { + accusation, + votes_for: vec![own_vote], + ec: ctx(), + }, + ); + id +} + +mod outcomes; +/// Digest computation must be deterministic for identical inputs and must +/// differ when any bound field changes. +mod voting; diff --git a/crates/slashing/src/accusation_voting/workflow_tests/outcomes.rs b/crates/slashing/src/accusation_voting/workflow_tests/outcomes.rs new file mode 100644 index 0000000000..5bd2eb0f8a --- /dev/null +++ b/crates/slashing/src/accusation_voting/workflow_tests/outcomes.rs @@ -0,0 +1,87 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +/// Agreeing votes that disagree on data_hash at quorum yield Equivocation. +#[test] +fn equivocation_when_hashes_differ_at_quorum() { + let me = signer(1); + let b = signer(2); + let accused = signer(9).address(); + let committee = vec![me.address(), b.address(), accused]; + let mut v = voting_with(&me, committee, 1, 2); + let sm = v.slashing_manager; + let data_hash_a = [0x11; 32]; + let data_hash_b = [0x22; 32]; + + let own = signed_vote(&me, sm, &v.e3_id, [0u8; 32], data_hash_a, NOW + VALIDITY); + let id = insert_pending(&mut v, &me, accused, data_hash_a, NOW + VALIDITY, own); + v.pending.get_mut(&id).unwrap().votes_for[0].accusation_id = id; + + // Voter b agrees but reports a different data_hash → equivocation. + let vote_b = signed_vote(&b, sm, &v.e3_id, id, data_hash_b, NOW + VALIDITY); + let actions = v.on_vote_received(vote_b, &ctx()); + let outcome = actions.iter().find_map(|a| match a { + VoteAction::PublishQuorum { quorum, .. } => Some(quorum.outcome.clone()), + _ => None, + }); + assert_eq!( + outcome, + Some(AccusationOutcome::Equivocation), + "differing data hashes at quorum must yield Equivocation" + ); +} + +/// Timeout below threshold yields Inconclusive; at/above threshold yields +/// AccusedFaulted. +#[test] +fn timeout_outcome_depends_on_vote_count() { + let me = signer(1); + let accused = signer(9).address(); + let committee = vec![me.address(), signer(2).address(), accused]; + let mut v = voting_with(&me, committee, 1, 2); + let sm = v.slashing_manager; + let data_hash = [0x11; 32]; + + let own = signed_vote(&me, sm, &v.e3_id, [0u8; 32], data_hash, NOW + VALIDITY); + let id = insert_pending(&mut v, &me, accused, data_hash, NOW + VALIDITY, own); + + // Only one agreeing vote, threshold is 2 → Inconclusive. + let (quorum, _ec) = v.on_vote_timeout(id).expect("timeout emits a decision"); + assert_eq!(quorum.outcome, AccusationOutcome::Inconclusive); + assert!(v.on_vote_timeout(id).is_none(), "second timeout is a no-op"); +} + +/// After a slash shrinks the live roster, ZK re-verification must still use the +/// canonical circuit committee size cached at construction. +#[test] +fn committee_size_unchanged_after_slash() { + let me = signer(1); + // Micro committee (T=4, N=9) — canonical pair in `from_threshold`. + let committee: Vec
= (1..=9u8).map(|b| signer(b).address()).collect(); + let mut v = voting_with(&me, committee.clone(), 4, 5); + + let slashed = committee[8]; + v.on_slash_executed(SlashExecuted { + e3_id: v.e3_id.clone(), + proposal_id: 1, + operator: slashed, + reason: [0u8; 32], + ticket_amount: 0, + license_amount: 0, + }); + assert_eq!(v.committee.len(), 8); + assert_eq!(v.committee_n, 9); + // Shrunken roster must not drive circuit resolution. + assert!( + CiphernodesCommitteeSize::from_threshold(v.circuit_threshold_t, v.committee.len()).is_err() + ); + assert_eq!( + CiphernodesCommitteeSize::from_threshold(v.circuit_threshold_t, v.committee_n).unwrap(), + CiphernodesCommitteeSize::Micro + ); +} diff --git a/crates/slashing/src/accusation_voting/workflow_tests/voting.rs b/crates/slashing/src/accusation_voting/workflow_tests/voting.rs new file mode 100644 index 0000000000..8b28071d2f --- /dev/null +++ b/crates/slashing/src/accusation_voting/workflow_tests/voting.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[test] +fn vote_digest_is_deterministic() { + let sm: Address = "0x5555555555555555555555555555555555555555" + .parse() + .unwrap(); + let voter: Address = "0x2222222222222222222222222222222222222222" + .parse() + .unwrap(); + let vote = AccusationVote { + e3_id: E3id::new("42", CHAIN_ID), + accusation_id: [0xab; 32], + voter, + data_hash: [0xcd; 32], + deadline: NOW, + signature: ArcBytes::default(), + }; + let a = AccusationVoting::vote_digest(&vote, sm); + let b = AccusationVoting::vote_digest(&vote, sm); + assert_eq!(a, b, "vote digest must be deterministic"); + + let mut vote2 = vote.clone(); + vote2.deadline = NOW + 1; + assert_ne!( + a, + AccusationVoting::vote_digest(&vote2, sm), + "changing deadline must change the digest" + ); +} + +/// A second agreeing vote that reaches `vote_quorum_h` must produce a single +/// AccusedFaulted quorum decision and remove the pending accusation. +#[test] +fn tally_reaches_quorum_at_threshold() { + let me = signer(1); + let b = signer(2); + let accused = signer(9).address(); + let committee = vec![me.address(), b.address(), accused]; + let mut v = voting_with(&me, committee, 1, 2); + let sm = v.slashing_manager; + let data_hash = [0x11; 32]; + + let own = signed_vote(&me, sm, &v.e3_id, [0u8; 32], data_hash, NOW + VALIDITY); + let id = insert_pending(&mut v, &me, accused, data_hash, NOW + VALIDITY, own); + // own vote's accusation_id was a placeholder; fix it to the real id. + v.pending.get_mut(&id).unwrap().votes_for[0].accusation_id = id; + + let vote_b = signed_vote(&b, sm, &v.e3_id, id, data_hash, NOW + VALIDITY); + let actions = v.on_vote_received(vote_b, &ctx()); + + let quorum = actions + .iter() + .filter_map(|a| match a { + VoteAction::PublishQuorum { quorum, .. } => Some(quorum), + _ => None, + }) + .count(); + assert_eq!(quorum, 1, "exactly one quorum decision expected"); + assert!( + !v.pending.contains_key(&id), + "pending accusation removed after quorum" + ); +} + +/// Inserting the same voter twice must not double-count nor re-trigger quorum. +#[test] +fn idempotent_vote_insert() { + let me = signer(1); + let b = signer(2); + let accused = signer(9).address(); + let committee = vec![me.address(), b.address(), accused]; + let mut v = voting_with(&me, committee, 1, 3); // quorum above what 2 votes reach + let sm = v.slashing_manager; + let data_hash = [0x11; 32]; + + let own = signed_vote(&me, sm, &v.e3_id, [0u8; 32], data_hash, NOW + VALIDITY); + let id = insert_pending(&mut v, &me, accused, data_hash, NOW + VALIDITY, own); + v.pending.get_mut(&id).unwrap().votes_for[0].accusation_id = id; + + let vote_b = signed_vote(&b, sm, &v.e3_id, id, data_hash, NOW + VALIDITY); + let _ = v.on_vote_received(vote_b.clone(), &ctx()); + let len_after_first = v.pending.get(&id).unwrap().votes_for.len(); + + // Same voter again — must be ignored. + let actions = v.on_vote_received(vote_b, &ctx()); + let len_after_second = v.pending.get(&id).unwrap().votes_for.len(); + assert_eq!( + len_after_first, len_after_second, + "duplicate voter must not be counted twice" + ); + assert!( + actions.is_empty(), + "duplicate vote must not emit any actions" + ); +} + +/// Quorum must trigger exactly at the M-th agreeing vote, not before. +#[test] +fn quorum_boundary() { + let me = signer(1); + let b = signer(2); + let c = signer(3); + let accused = signer(9).address(); + let committee = vec![me.address(), b.address(), c.address(), accused]; + let mut v = voting_with(&me, committee, 1, 3); + let sm = v.slashing_manager; + let data_hash = [0x11; 32]; + + let own = signed_vote(&me, sm, &v.e3_id, [0u8; 32], data_hash, NOW + VALIDITY); + let id = insert_pending(&mut v, &me, accused, data_hash, NOW + VALIDITY, own); + v.pending.get_mut(&id).unwrap().votes_for[0].accusation_id = id; + + // 2nd vote — below threshold of 3, no quorum. + let vote_b = signed_vote(&b, sm, &v.e3_id, id, data_hash, NOW + VALIDITY); + let actions = v.on_vote_received(vote_b, &ctx()); + assert!( + actions.is_empty(), + "no quorum before reaching threshold M=3" + ); + assert!(v.pending.contains_key(&id)); + + // 3rd vote — reaches threshold, quorum fires. + let vote_c = signed_vote(&c, sm, &v.e3_id, id, data_hash, NOW + VALIDITY); + let actions = v.on_vote_received(vote_c, &ctx()); + assert!( + actions + .iter() + .any(|a| matches!(a, VoteAction::PublishQuorum { .. })), + "quorum must fire at the M-th vote" + ); +} diff --git a/crates/slashing/src/actors/mod.rs b/crates/slashing/src/actors.rs similarity index 57% rename from crates/slashing/src/actors/mod.rs rename to crates/slashing/src/actors.rs index 5586d64a65..a3e5a41291 100644 --- a/crates/slashing/src/actors/mod.rs +++ b/crates/slashing/src/actors.rs @@ -4,10 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -//! Thin actix shells that translate [`InterfoldEvent`]s into pure-domain calls -//! and perform the I/O those domain services request. -//! -//! [`InterfoldEvent`]: e3_events::InterfoldEvent +//! Compatibility view of thin actors stored with their slashing capabilities. +#[path = "accusation_voting/actor.rs"] pub mod accusation_manager; +#[path = "commitment_consistency/actor.rs"] pub mod commitment_consistency_checker; diff --git a/crates/slashing/src/actors/accusation_manager.rs b/crates/slashing/src/actors/accusation_manager.rs deleted file mode 100644 index 7df032e82f..0000000000 --- a/crates/slashing/src/actors/accusation_manager.rs +++ /dev/null @@ -1,714 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Off-chain accusation quorum protocol for fault attribution. -//! -//! When a node detects a ZK proof failure from another committee member, it -//! broadcasts a [`ProofFailureAccusation`] over gossip. Other committee members -//! independently check the same proof and respond with [`AccusationVote`]s. -//! Once the on-chain quorum `H` votes is reached, the -//! actor emits [`AccusationQuorumReached`] for downstream consumers (aggregator -//! exclusion, on-chain slash submission). -//! -//! ## Architecture -//! -//! This file is a **thin actix shell**. All protocol logic lives in the plain, -//! synchronous [`AccusationVoting`] service ([`crate::accusation_voting`]). The -//! actor's only job is to translate inbound [`InterfoldEvent`]s into service -//! calls and to perform the I/O ([`VoteAction`]s) the service returns — -//! publishing gossip events, dispatching ZK requests, and managing vote -//! timeouts. -//! -//! ## Proof-type-specific behavior -//! -//! | Proof | Attestation | Notes | -//! |---------|----------------------------|--------------------------------------------| -//! | C0 | All nodes independently | Everyone receives via DHT | -//! | C1 | All nodes independently | Bundled in ThresholdShareCreated | -//! | C2a/C2b | All nodes independently | Same proof bytes for all recipients | -//! | C3a/C3b | Forwarding required | Per-recipient; accuser forwards payload | -//! | C4 | All nodes independently | Broadcast via gossip | -//! | C5 | Committee attests | Aggregator-generated; nodes verify off-chain| -//! | C6 | All nodes independently | Broadcast via gossip | -//! | C7 | On-chain verification | Not handled here (on-chain verifier) | - -use std::collections::HashMap; -use std::sync::Arc; -use std::time::{SystemTime, UNIX_EPOCH}; - -use actix::{Actor, Addr, AsyncContext, Context, Handler, SpawnHandle}; -use alloy::primitives::{Address, Bytes}; -use alloy::signers::local::PrivateKeySigner; -use e3_events::{ - AccusationVote, BusHandle, CommitmentConsistencyViolation, ComputeRequestError, - ComputeResponse, E3id, EventPublisher, EventSubscriber, EventType, InterfoldEvent, - InterfoldEventData, ProofFailureAccusation, ProofType, ProofVerificationFailed, - ProofVerificationPassed, TypedEvent, -}; -use e3_utils::NotifySync; -use e3_zk_helpers::CiphernodesCommitteeSize; -use tracing::{error, warn}; - -use crate::domain::accusation_voting::{AccusationVoting, VoteAction}; - -/// Abstraction over wall-clock time so the deadline-stamping logic is -/// deterministically testable. Production uses [`SystemClock`], which reads -/// `SystemTime::now()`; tests can inject a mock clock that returns fixed -/// timestamps. -pub trait Clock: Send + Sync + 'static { - /// Current Unix time in seconds. Returns `0` if the platform clock is - /// pre-`UNIX_EPOCH` (a broken clock should not silently produce - /// signatures that look valid forever — the on-chain check will then - /// reject the resulting deadline immediately). - fn unix_now_secs(&self) -> u64; -} - -/// Production clock backed by `SystemTime::now()`. -pub struct SystemClock; - -impl Clock for SystemClock { - fn unix_now_secs(&self) -> u64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0) - } -} - -/// Thin actix shell around the [`AccusationVoting`] domain service. -/// -/// **Lifecycle**: One instance per E3 computation. Created by -/// [`AccusationManagerExtension`] when [`CommitteeFinalized`] fires and -/// destroyed when the E3 completes or the node shuts down. All protocol state -/// lives inside the owned [`AccusationVoting`] service and is therefore -/// naturally scoped to a single E3. -/// -/// **Ephemeral**: This actor does *not* persist state across restarts. -/// In-flight accusations are lost on node restart (accepted trade-off: -/// they would have timed out within the vote timeout anyway). -/// -/// Subscribes to: -/// - [`ProofVerificationFailed`] — local proof failure detection -/// - [`ProofVerificationPassed`] — cache successful verification for voting -/// - [`ProofFailureAccusation`] — incoming accusations from other nodes via gossip -/// - [`AccusationVote`] — incoming votes from other nodes via gossip -/// - [`SlashExecuted`] — on-chain slash confirmation for committee updates -/// -/// Publishes: -/// - [`ProofFailureAccusation`] — broadcast own accusations via gossip -/// - [`AccusationVote`] — broadcast own votes via gossip -/// - [`AccusationQuorumReached`] — quorum decision for downstream consumers -/// -/// [`AccusationManagerExtension`]: crate::accusation_manager_ext::AccusationManagerExtension -/// [`CommitteeFinalized`]: e3_events::CommitteeFinalized -/// [`AccusationQuorumReached`]: e3_events::AccusationQuorumReached -/// [`SlashExecuted`]: e3_events::SlashExecuted -pub struct AccusationManager { - bus: BusHandle, - /// Plain, synchronous protocol core. Owns all accusation/vote state. - voting: AccusationVoting, - /// Active vote-collection timeouts keyed by accusation_id. Managed entirely - /// by the actor — the service only signals start/cancel via [`VoteAction`]. - timeout_handles: HashMap<[u8; 32], SpawnHandle>, -} - -impl AccusationManager { - fn canonical_vote_quorum(circuit_threshold_t: usize, committee_n: usize) -> usize { - match CiphernodesCommitteeSize::from_threshold(circuit_threshold_t, committee_n) { - Ok(size) => size.values().h, - Err(err) => { - // Preserve the historical constructor behavior for external callers with a - // non-canonical test committee. Production creation is validated by the - // extension and uses `setup_with_quorum` below. - warn!( - circuit_threshold_t, - committee_n, - error = %err, - "Unknown committee size; falling back to the supplied threshold as vote quorum" - ); - circuit_threshold_t - } - } - } - - /// Construct an actor with the production [`SystemClock`]. Use - /// [`AccusationManager::new_with_clock`] in tests that need deterministic - /// timestamps. - #[allow(clippy::too_many_arguments)] - pub fn new( - bus: &BusHandle, - e3_id: E3id, - signer: PrivateKeySigner, - slashing_manager: Address, - committee: Vec
, - circuit_threshold_t: usize, - vote_validity_secs: u64, - accusation_deadline_skew_secs: u64, - params_preset: e3_fhe_params::BfvPreset, - ) -> Self { - Self::new_with_clock( - bus, - e3_id, - signer, - slashing_manager, - committee, - circuit_threshold_t, - vote_validity_secs, - accusation_deadline_skew_secs, - params_preset, - Arc::new(SystemClock), - ) - } - - /// Construct an actor with an explicit [`Clock`]. Allows unit tests to - /// drive deadline computation without touching wall-clock time. - #[allow(clippy::too_many_arguments)] - pub fn new_with_clock( - bus: &BusHandle, - e3_id: E3id, - signer: PrivateKeySigner, - slashing_manager: Address, - committee: Vec
, - circuit_threshold_t: usize, - vote_validity_secs: u64, - accusation_deadline_skew_secs: u64, - params_preset: e3_fhe_params::BfvPreset, - clock: Arc, - ) -> Self { - let vote_quorum_h = Self::canonical_vote_quorum(circuit_threshold_t, committee.len()); - Self::new_with_clock_and_quorum( - bus, - e3_id, - signer, - slashing_manager, - committee, - circuit_threshold_t, - vote_quorum_h, - vote_validity_secs, - accusation_deadline_skew_secs, - params_preset, - clock, - ) - } - - #[allow(clippy::too_many_arguments)] - fn new_with_clock_and_quorum( - bus: &BusHandle, - e3_id: E3id, - signer: PrivateKeySigner, - slashing_manager: Address, - committee: Vec
, - circuit_threshold_t: usize, - vote_quorum_h: usize, - vote_validity_secs: u64, - accusation_deadline_skew_secs: u64, - params_preset: e3_fhe_params::BfvPreset, - clock: Arc, - ) -> Self { - Self { - bus: bus.clone(), - voting: AccusationVoting::new( - e3_id, - signer, - slashing_manager, - committee, - circuit_threshold_t, - vote_quorum_h, - vote_validity_secs, - accusation_deadline_skew_secs, - params_preset, - clock, - ), - timeout_handles: HashMap::new(), - } - } - - #[allow(clippy::too_many_arguments)] - pub fn setup( - bus: &BusHandle, - e3_id: E3id, - signer: PrivateKeySigner, - slashing_manager: Address, - committee: Vec
, - circuit_threshold_t: usize, - vote_validity_secs: u64, - accusation_deadline_skew_secs: u64, - params_preset: e3_fhe_params::BfvPreset, - ) -> Addr { - let vote_quorum_h = Self::canonical_vote_quorum(circuit_threshold_t, committee.len()); - Self::setup_with_quorum( - bus, - e3_id, - signer, - slashing_manager, - committee, - circuit_threshold_t, - vote_quorum_h, - vote_validity_secs, - accusation_deadline_skew_secs, - params_preset, - ) - } - - #[allow(clippy::too_many_arguments)] - pub(crate) fn setup_with_quorum( - bus: &BusHandle, - e3_id: E3id, - signer: PrivateKeySigner, - slashing_manager: Address, - committee: Vec
, - circuit_threshold_t: usize, - vote_quorum_h: usize, - vote_validity_secs: u64, - accusation_deadline_skew_secs: u64, - params_preset: e3_fhe_params::BfvPreset, - ) -> Addr { - let addr = Self::new_with_clock_and_quorum( - bus, - e3_id, - signer, - slashing_manager, - committee, - circuit_threshold_t, - vote_quorum_h, - vote_validity_secs, - accusation_deadline_skew_secs, - params_preset, - Arc::new(SystemClock), - ) - .start(); - bus.subscribe(EventType::ProofVerificationFailed, addr.clone().into()); - bus.subscribe(EventType::ProofVerificationPassed, addr.clone().into()); - bus.subscribe(EventType::ProofFailureAccusation, addr.clone().into()); - bus.subscribe(EventType::AccusationVote, addr.clone().into()); - bus.subscribe(EventType::ComputeResponse, addr.clone().into()); - bus.subscribe(EventType::ComputeRequestError, addr.clone().into()); - bus.subscribe(EventType::SlashExecuted, addr.clone().into()); - bus.subscribe( - EventType::CommitmentConsistencyViolation, - addr.clone().into(), - ); - addr - } - - /// Canonical EIP-712 typed-data hash for a vote. - /// - /// Delegates to [`AccusationVoting::vote_digest`]. Exposed `pub` so the - /// Anvil parity test in - /// `crates/zk-prover/tests/slashing_integration_tests.rs` can sign votes - /// through the **same** code path the production actor uses. - pub fn vote_digest(vote: &AccusationVote, verifying_contract: Address) -> [u8; 32] { - AccusationVoting::vote_digest(vote, verifying_contract) - } - - /// Cache a successful proof verification result for a specific - /// (accused, proof_type). Allows the node to vote on accusations from - /// other nodes. - pub fn cache_verification_result( - &mut self, - accused: Address, - proof_type: ProofType, - data_hash: [u8; 32], - passed: bool, - evidence: Bytes, - ) { - self.voting - .cache_verification_result(accused, proof_type, data_hash, passed, evidence); - } - - /// Perform the I/O the [`AccusationVoting`] service requested. - /// - /// This is the *only* place the actor publishes events or touches timers — - /// keeping all protocol decisions in the pure service. - fn apply_actions(&mut self, actions: Vec, ctx: &mut Context) { - for action in actions { - match action { - VoteAction::PublishAccusation { - accusation, - ec, - dedup_key, - } => { - if let Err(err) = self.bus.publish(accusation, ec) { - error!("Failed to broadcast ProofFailureAccusation: {err}"); - // Preserve the original rollback: re-allow this - // (accused, proof_type) accusation on a dead bus. - self.voting.rollback_initiation(&dedup_key); - } - } - VoteAction::PublishVote { vote, ec } => { - if let Err(err) = self.bus.publish(vote, ec) { - error!("Failed to broadcast AccusationVote: {err}"); - } - } - VoteAction::PublishQuorum { quorum, ec } => { - if let Err(err) = self.bus.publish(quorum, ec) { - error!("Failed to publish AccusationQuorumReached: {err}"); - } - } - VoteAction::DispatchZk { - request, - ec, - correlation_id, - } => { - if let Err(err) = self.bus.publish(request, ec) { - error!("Failed to dispatch C3a/C3b ZK re-verification: {err}"); - self.voting.discard_reverification(&correlation_id); - } - } - VoteAction::StartTimeout(accusation_id) => { - let timeout = self.voting.vote_timeout(); - let handle = ctx.run_later(timeout, move |act, _ctx| { - act.timeout_handles.remove(&accusation_id); - if let Some((quorum, ec)) = act.voting.on_vote_timeout(accusation_id) { - if let Err(err) = act.bus.publish(quorum, ec) { - error!( - "Failed to publish AccusationQuorumReached on timeout: {err}" - ); - } - } - }); - self.timeout_handles.insert(accusation_id, handle); - } - VoteAction::CancelTimeout(accusation_id) => { - if let Some(handle) = self.timeout_handles.remove(&accusation_id) { - ctx.cancel_future(handle); - } - } - } - } - } -} - -impl Actor for AccusationManager { - type Context = Context; -} - -impl Handler for AccusationManager { - type Result = (); - - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - match msg { - InterfoldEventData::ProofVerificationFailed(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ProofVerificationPassed(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ProofFailureAccusation(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::AccusationVote(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ComputeResponse(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ComputeRequestError(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::SlashExecuted(data) => { - self.voting.on_slash_executed(data); - } - InterfoldEventData::CommitmentConsistencyViolation(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - _ => (), - } - } -} - -impl Handler> for AccusationManager { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - let (data, ec) = msg.into_components(); - let actions = self.voting.on_local_proof_failure(data, &ec); - self.apply_actions(actions, ctx); - } -} - -impl Handler> for AccusationManager { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - let (data, _ec) = msg.into_components(); - self.voting.on_proof_verification_passed(data); - } -} - -impl Handler> for AccusationManager { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - let (data, ec) = msg.into_components(); - let actions = self.voting.on_accusation_received(data, &ec); - self.apply_actions(actions, ctx); - } -} - -impl Handler> for AccusationManager { - type Result = (); - - fn handle(&mut self, msg: TypedEvent, ctx: &mut Self::Context) -> Self::Result { - let (data, ec) = msg.into_components(); - let actions = self.voting.on_vote_received(data, &ec); - self.apply_actions(actions, ctx); - } -} - -impl Handler> for AccusationManager { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - let actions = self.voting.handle_reverification_response(msg); - self.apply_actions(actions, ctx); - } -} - -impl Handler> for AccusationManager { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.voting.handle_reverification_error(msg); - } -} - -impl Handler> for AccusationManager { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - let (data, ec) = msg.into_components(); - let actions = self.voting.on_consistency_violation(data, &ec); - self.apply_actions(actions, ctx); - } -} - -// ════════════════════════════════════════════════════════════════════════════ -// Tests -// ════════════════════════════════════════════════════════════════════════════ -// -// These tests pin the actor's public EIP-712 digest computation (delegated to -// `AccusationVoting`) to the exact bytes that off-chain test helpers (and -// ultimately the on-chain `SlashingManager._verifyVotes`) expect. - -#[cfg(test)] -mod tests { - use super::*; - use alloy::primitives::{keccak256, FixedBytes, U256}; - use alloy::signers::SignerSync; - use alloy::sol_types::SolValue; - use e3_events::{VOTE_DOMAIN_NAME, VOTE_DOMAIN_VERSION, VOTE_TYPEHASH_STR}; - use e3_utils::ArcBytes; - - /// Default clock-skew allowance when validating peer-stamped accusation - /// deadlines (mirrors the production extension default). - const DEFAULT_ACCUSATION_DEADLINE_SKEW_SECS: u64 = 30; - - /// Independent re-derivation of the EIP-712 vote digest, mirroring exactly - /// what `SlashingManager._verifyVotes` computes on chain. - fn reference_vote_digest( - chain_id: u64, - verifying_contract: Address, - e3_id: u64, - accusation_id: [u8; 32], - voter: Address, - data_hash: [u8; 32], - deadline: u64, - ) -> [u8; 32] { - let domain_typehash: [u8; 32] = keccak256( - "EIP712Domain(string name,string version,uint256 chainId,address verifyingContract)", - ) - .into(); - let name_hash: [u8; 32] = keccak256(VOTE_DOMAIN_NAME).into(); - let version_hash: [u8; 32] = keccak256(VOTE_DOMAIN_VERSION).into(); - let domain_separator: [u8; 32] = keccak256( - ( - domain_typehash, - name_hash, - version_hash, - U256::from(chain_id), - verifying_contract, - ) - .abi_encode(), - ) - .into(); - - let typehash: [u8; 32] = keccak256(VOTE_TYPEHASH_STR).into(); - let struct_hash: [u8; 32] = keccak256( - ( - typehash, - U256::from(e3_id), - FixedBytes::<32>::from(accusation_id), - voter, - FixedBytes::<32>::from(data_hash), - U256::from(deadline), - ) - .abi_encode(), - ) - .into(); - - let mut buf = Vec::with_capacity(2 + 32 + 32); - buf.push(0x19); - buf.push(0x01); - buf.extend_from_slice(&domain_separator); - buf.extend_from_slice(&struct_hash); - keccak256(&buf).into() - } - - /// The actor's `vote_digest` must equal the reference digest byte-for-byte. - #[test] - fn vote_digest_matches_reference() { - let chain_id = 31337u64; - let verifying_contract: Address = "0x9999999999999999999999999999999999999999" - .parse() - .unwrap(); - let voter: Address = "0x2222222222222222222222222222222222222222" - .parse() - .unwrap(); - let accusation_id = [0xab; 32]; - let data_hash = [0xcd; 32]; - let deadline: u64 = 1_700_000_000; - - let vote = AccusationVote { - e3_id: E3id::new("42", chain_id), - accusation_id, - voter, - data_hash, - deadline, - signature: ArcBytes::default(), - }; - - let actor = AccusationManager::vote_digest(&vote, verifying_contract); - let reference = reference_vote_digest( - chain_id, - verifying_contract, - 42, - accusation_id, - voter, - data_hash, - deadline, - ); - - assert_eq!( - actor, reference, - "AccusationManager::vote_digest drifted from the reference EIP-712 \ - computation. Check VOTE_TYPEHASH_STR / VOTE_DOMAIN_NAME against \ - SlashingManager.sol — these MUST stay byte-equal across crates." - ); - } - - /// Sign-and-recover round-trip using the actor's digest. - #[test] - fn actor_signature_recovers_to_voter() { - let signer: PrivateKeySigner = - "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" - .parse() - .unwrap(); - let voter = signer.address(); - let verifying_contract: Address = "0x5555555555555555555555555555555555555555" - .parse() - .unwrap(); - let chain_id = 31337u64; - - let vote = AccusationVote { - e3_id: E3id::new("12345", chain_id), - accusation_id: [0x07; 32], - voter, - data_hash: [0x08; 32], - deadline: 1_700_000_000, - signature: ArcBytes::default(), - }; - - let digest = AccusationManager::vote_digest(&vote, verifying_contract); - let sig = signer - .sign_hash_sync(&FixedBytes::<32>::from(digest)) - .unwrap(); - let recovered = sig - .recover_address_from_prehash(&FixedBytes::<32>::from(digest)) - .expect("recover"); - assert_eq!( - recovered, voter, - "signing the actor's digest and recovering must yield the voter" - ); - } - - /// The accusation digest must include `deadline`. - #[test] - fn accusation_digest_binds_deadline() { - let make = |deadline: u64| ProofFailureAccusation { - e3_id: E3id::new("9", 31337), - accuser: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - .parse() - .unwrap(), - accused: "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" - .parse() - .unwrap(), - accused_party_id: 1, - proof_type: ProofType::C1PkGeneration, - data_hash: [0x42; 32], - deadline, - signed_payload: None, - signature: ArcBytes::default(), - }; - let a = AccusationVoting::accusation_digest(&make(1_700_000_000)); - let b = AccusationVoting::accusation_digest(&make(1_700_000_001)); - assert_ne!(a, b, "deadline must be part of the accusation digest"); - } - - #[test] - fn peer_deadline_acceptance_enforces_local_window() { - let now = 1_700_000_000u64; - let validity = 1_800u64; - let skew = DEFAULT_ACCUSATION_DEADLINE_SKEW_SECS; - let max_ok = now + validity + skew; - - assert!( - !AccusationVoting::is_peer_deadline_acceptable(now, now, validity, skew), - "deadline equal to now must be rejected" - ); - assert!( - !AccusationVoting::is_peer_deadline_acceptable(now - 1, now, validity, skew), - "expired deadline must be rejected" - ); - assert!( - AccusationVoting::is_peer_deadline_acceptable(max_ok, now, validity, skew), - "deadline at upper bound must be accepted" - ); - assert!( - !AccusationVoting::is_peer_deadline_acceptable(max_ok + 1, now, validity, skew), - "far-future deadline must be rejected" - ); - assert!( - !AccusationVoting::is_peer_deadline_acceptable(now + 10, now, 0, skew), - "vote_validity_secs=0 must reject peer accusations" - ); - } -} diff --git a/crates/slashing/src/actors/commitment_consistency_checker.rs b/crates/slashing/src/commitment_consistency/actor.rs similarity index 100% rename from crates/slashing/src/actors/commitment_consistency_checker.rs rename to crates/slashing/src/commitment_consistency/actor.rs diff --git a/crates/slashing/src/domain/commitment_consistency.rs b/crates/slashing/src/commitment_consistency/workflow.rs similarity index 64% rename from crates/slashing/src/domain/commitment_consistency.rs rename to crates/slashing/src/commitment_consistency/workflow.rs index 2d70035213..1d0c53e162 100644 --- a/crates/slashing/src/domain/commitment_consistency.rs +++ b/crates/slashing/src/commitment_consistency/workflow.rs @@ -13,7 +13,7 @@ //! - evaluating registered [`CommitmentLink`]s across the three [`LinkScope`]s //! - building the evidence preimage for [`CommitmentConsistencyViolation`]s //! -//! Following the same pattern as [`crate::domain::accusation_voting`], the +//! Following the same pattern as [`crate::workflow::accusation_voting`], the //! [`CommitmentConsistency`] service owns the protocol state and exposes plain //! methods that mutate that state and **return decisions/data** (violations to //! emit, the pre-ZK completion message). The service itself performs **no** @@ -437,319 +437,5 @@ impl CommitmentConsistency { // ════════════════════════════════════════════════════════════════════════════ #[cfg(test)] -mod tests { - use super::*; - use e3_events::{CorrelationId, FieldValue, PartyProofData, VerificationKind}; - - /// A minimal same-party commitment link: extracts the first 32 bytes of the - /// source public signals and requires them to equal the first 32 bytes of - /// the target public signals. - struct TestLink { - scope: LinkScope, - source: ProofType, - target: ProofType, - } - - impl CommitmentLink for TestLink { - fn name(&self) -> &'static str { - "test_link" - } - fn source_proof_type(&self) -> ProofType { - self.source - } - fn target_proof_type(&self) -> ProofType { - self.target - } - fn scope(&self) -> LinkScope { - self.scope - } - fn extract_source_values(&self, public_signals: &[u8]) -> Vec { - if public_signals.len() < 32 { - return Vec::new(); - } - let mut v = [0u8; 32]; - v.copy_from_slice(&public_signals[..32]); - vec![v] - } - fn check_signals( - &self, - source_values: &[FieldValue], - target_public_signals: &[u8], - ) -> bool { - if target_public_signals.len() < 32 { - return false; - } - source_values - .iter() - .any(|v| v[..] == target_public_signals[..32]) - } - } - - fn e3() -> E3id { - E3id::new("7", 31337) - } - - fn addr(byte: u8) -> Address { - Address::from([byte; 20]) - } - - fn signals(byte: u8) -> ArcBytes { - ArcBytes::from_bytes(&[byte; 32]) - } - - fn passed( - e3_id: E3id, - party_id: u64, - address: Address, - proof_type: ProofType, - data_hash: [u8; 32], - public_signals: ArcBytes, - ) -> ProofVerificationPassed { - ProofVerificationPassed { - e3_id, - party_id, - address, - proof_type, - data_hash, - public_signals, - proof_data: ArcBytes::from_bytes(&[0xAA, 0xBB]), - } - } - - fn same_party_link() -> Box { - Box::new(TestLink { - scope: LinkScope::SameParty, - source: ProofType::C1PkGeneration, - target: ProofType::C2aSkShareComputation, - }) - } - - #[test] - fn consistent_same_party_proofs_emit_no_violation() { - let mut svc = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); - let a = addr(1); - - // Target first (C2) so the source check has something to compare to. - let v = svc.on_proof_verified(passed( - e3(), - 1, - a, - ProofType::C2aSkShareComputation, - [0x11; 32], - signals(0x42), - )); - assert!(v.is_empty()); - - // Source (C1) with matching signals — consistent. - let v = svc.on_proof_verified(passed( - e3(), - 1, - a, - ProofType::C1PkGeneration, - [0x22; 32], - signals(0x42), - )); - assert!(v.is_empty(), "matching commitments must not violate"); - } - - #[test] - fn mismatched_same_party_proofs_emit_violation() { - let mut svc = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); - let a = addr(2); - - svc.on_proof_verified(passed( - e3(), - 3, - a, - ProofType::C2aSkShareComputation, - [0x11; 32], - signals(0x01), - )); - - let v = svc.on_proof_verified(passed( - e3(), - 3, - a, - ProofType::C1PkGeneration, - [0x22; 32], - signals(0x99), - )); - - assert_eq!( - v.len(), - 1, - "mismatched commitments must produce a violation" - ); - let viol = &v[0]; - assert_eq!(viol.accused_party_id, 3); - assert_eq!(viol.accused_address, a); - assert_eq!(viol.proof_type, ProofType::C1PkGeneration); - assert_eq!(viol.data_hash, [0x22; 32]); - assert!( - !viol.evidence.is_empty(), - "evidence preimage must be present" - ); - } - - #[test] - fn zero_data_hash_mismatch_is_skipped() { - let mut svc = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); - let a = addr(3); - - svc.on_proof_verified(passed( - e3(), - 4, - a, - ProofType::C2aSkShareComputation, - [0x11; 32], - signals(0x01), - )); - - // Source carries an unresolved (zero) data_hash — must be skipped. - let v = svc.on_proof_verified(passed( - e3(), - 4, - a, - ProofType::C1PkGeneration, - [0u8; 32], - signals(0x99), - )); - - assert!(v.is_empty(), "zero-data_hash mismatch must be skipped"); - } - - #[test] - fn foreign_e3_id_is_ignored() { - let mut svc = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); - let other = E3id::new("999", 31337); - let a = addr(4); - - let v = svc.on_proof_verified(passed( - other.clone(), - 1, - a, - ProofType::C1PkGeneration, - [0x22; 32], - signals(0x99), - )); - assert!(v.is_empty(), "proofs for a foreign E3 must be ignored"); - - let req = CommitmentConsistencyCheckRequested { - e3_id: other, - kind: VerificationKind::ShareProofs, - correlation_id: CorrelationId::new(), - party_proofs: vec![], - }; - assert!( - svc.on_check_requested(req).is_none(), - "pre-ZK requests for a foreign E3 must return None" - ); - } - - #[test] - fn pre_zk_check_flags_and_evicts_inconsistent_party() { - let mut svc = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); - let honest = addr(5); - let faulty = addr(6); - - let req = CommitmentConsistencyCheckRequested { - e3_id: e3(), - kind: VerificationKind::ShareProofs, - correlation_id: CorrelationId::new(), - party_proofs: vec![ - PartyProofData { - party_id: 1, - address: honest, - proofs: vec![ - ( - ProofType::C1PkGeneration, - signals(0x42), - [0xa1; 32], - ArcBytes::from_bytes(&[0x01]), - ), - ( - ProofType::C2aSkShareComputation, - signals(0x42), - [0xa2; 32], - ArcBytes::from_bytes(&[0x02]), - ), - ], - }, - PartyProofData { - party_id: 2, - address: faulty, - proofs: vec![ - ( - ProofType::C1PkGeneration, - signals(0x11), - [0xb1; 32], - ArcBytes::from_bytes(&[0x03]), - ), - ( - ProofType::C2aSkShareComputation, - signals(0x99), - [0xb2; 32], - ArcBytes::from_bytes(&[0x04]), - ), - ], - }, - ], - }; - - let outcome = svc.on_check_requested(req).expect("same e3"); - assert!( - outcome.complete.inconsistent_parties.contains(&2), - "faulty party must be flagged" - ); - assert!( - !outcome.complete.inconsistent_parties.contains(&1), - "honest party must not be flagged" - ); - assert_eq!(outcome.violations.len(), 1); - assert_eq!(outcome.violations[0].accused_party_id, 2); - - // The faulty party's cache entries are evicted, so a later post-ZK - // event for the honest party does not re-report the faulty one. - let v = svc.on_proof_verified(passed( - e3(), - 1, - honest, - ProofType::C1PkGeneration, - [0xa1; 32], - signals(0x42), - )); - assert!(v.is_empty(), "evicted faulty party must not resurface"); - } - - #[test] - fn c2_sender_at_or_above_h_skips_c4_cross_check() { - let link = Box::new(TestLink { - scope: LinkScope::SourceMustExistInTargets, - source: ProofType::C2aSkShareComputation, - target: ProofType::C4aSkShareDecryption, - }); - let mut svc = CommitmentConsistency::new(e3(), vec![link], 2); - - // Party 2 (>= H) C2 cannot appear in C4 rows; must not be faulted. - svc.on_proof_verified(passed( - e3(), - 2, - addr(0x22), - ProofType::C2aSkShareComputation, - [0xc2; 32], - signals(0x22), - )); - let violations = svc.on_proof_verified(passed( - e3(), - 1, - addr(0x11), - ProofType::C4aSkShareDecryption, - [0xc4; 32], - signals(0x11), - )); - assert!( - violations.is_empty(), - "party_id >= H must be outside C4 expected_commitments roster" - ); - } -} +#[path = "workflow_tests.rs"] +mod tests; diff --git a/crates/slashing/src/commitment_consistency/workflow_tests.rs b/crates/slashing/src/commitment_consistency/workflow_tests.rs new file mode 100644 index 0000000000..29be2b3752 --- /dev/null +++ b/crates/slashing/src/commitment_consistency/workflow_tests.rs @@ -0,0 +1,316 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use e3_events::{CorrelationId, FieldValue, PartyProofData, VerificationKind}; + +/// A minimal same-party commitment link: extracts the first 32 bytes of the +/// source public signals and requires them to equal the first 32 bytes of +/// the target public signals. +struct TestLink { + scope: LinkScope, + source: ProofType, + target: ProofType, +} + +impl CommitmentLink for TestLink { + fn name(&self) -> &'static str { + "test_link" + } + fn source_proof_type(&self) -> ProofType { + self.source + } + fn target_proof_type(&self) -> ProofType { + self.target + } + fn scope(&self) -> LinkScope { + self.scope + } + fn extract_source_values(&self, public_signals: &[u8]) -> Vec { + if public_signals.len() < 32 { + return Vec::new(); + } + let mut v = [0u8; 32]; + v.copy_from_slice(&public_signals[..32]); + vec![v] + } + fn check_signals(&self, source_values: &[FieldValue], target_public_signals: &[u8]) -> bool { + if target_public_signals.len() < 32 { + return false; + } + source_values + .iter() + .any(|v| v[..] == target_public_signals[..32]) + } +} + +fn e3() -> E3id { + E3id::new("7", 31337) +} + +fn addr(byte: u8) -> Address { + Address::from([byte; 20]) +} + +fn signals(byte: u8) -> ArcBytes { + ArcBytes::from_bytes(&[byte; 32]) +} + +fn passed( + e3_id: E3id, + party_id: u64, + address: Address, + proof_type: ProofType, + data_hash: [u8; 32], + public_signals: ArcBytes, +) -> ProofVerificationPassed { + ProofVerificationPassed { + e3_id, + party_id, + address, + proof_type, + data_hash, + public_signals, + proof_data: ArcBytes::from_bytes(&[0xAA, 0xBB]), + } +} + +fn same_party_link() -> Box { + Box::new(TestLink { + scope: LinkScope::SameParty, + source: ProofType::C1PkGeneration, + target: ProofType::C2aSkShareComputation, + }) +} + +#[test] +fn consistent_same_party_proofs_emit_no_violation() { + let mut svc = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); + let a = addr(1); + + // Target first (C2) so the source check has something to compare to. + let v = svc.on_proof_verified(passed( + e3(), + 1, + a, + ProofType::C2aSkShareComputation, + [0x11; 32], + signals(0x42), + )); + assert!(v.is_empty()); + + // Source (C1) with matching signals — consistent. + let v = svc.on_proof_verified(passed( + e3(), + 1, + a, + ProofType::C1PkGeneration, + [0x22; 32], + signals(0x42), + )); + assert!(v.is_empty(), "matching commitments must not violate"); +} + +#[test] +fn mismatched_same_party_proofs_emit_violation() { + let mut svc = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); + let a = addr(2); + + svc.on_proof_verified(passed( + e3(), + 3, + a, + ProofType::C2aSkShareComputation, + [0x11; 32], + signals(0x01), + )); + + let v = svc.on_proof_verified(passed( + e3(), + 3, + a, + ProofType::C1PkGeneration, + [0x22; 32], + signals(0x99), + )); + + assert_eq!( + v.len(), + 1, + "mismatched commitments must produce a violation" + ); + let viol = &v[0]; + assert_eq!(viol.accused_party_id, 3); + assert_eq!(viol.accused_address, a); + assert_eq!(viol.proof_type, ProofType::C1PkGeneration); + assert_eq!(viol.data_hash, [0x22; 32]); + assert!( + !viol.evidence.is_empty(), + "evidence preimage must be present" + ); +} + +#[test] +fn zero_data_hash_mismatch_is_skipped() { + let mut svc = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); + let a = addr(3); + + svc.on_proof_verified(passed( + e3(), + 4, + a, + ProofType::C2aSkShareComputation, + [0x11; 32], + signals(0x01), + )); + + // Source carries an unresolved (zero) data_hash — must be skipped. + let v = svc.on_proof_verified(passed( + e3(), + 4, + a, + ProofType::C1PkGeneration, + [0u8; 32], + signals(0x99), + )); + + assert!(v.is_empty(), "zero-data_hash mismatch must be skipped"); +} + +#[test] +fn foreign_e3_id_is_ignored() { + let mut svc = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); + let other = E3id::new("999", 31337); + let a = addr(4); + + let v = svc.on_proof_verified(passed( + other.clone(), + 1, + a, + ProofType::C1PkGeneration, + [0x22; 32], + signals(0x99), + )); + assert!(v.is_empty(), "proofs for a foreign E3 must be ignored"); + + let req = CommitmentConsistencyCheckRequested { + e3_id: other, + kind: VerificationKind::ShareProofs, + correlation_id: CorrelationId::new(), + party_proofs: vec![], + }; + assert!( + svc.on_check_requested(req).is_none(), + "pre-ZK requests for a foreign E3 must return None" + ); +} + +#[test] +fn pre_zk_check_flags_and_evicts_inconsistent_party() { + let mut svc = CommitmentConsistency::new(e3(), vec![same_party_link()], 2); + let honest = addr(5); + let faulty = addr(6); + + let req = CommitmentConsistencyCheckRequested { + e3_id: e3(), + kind: VerificationKind::ShareProofs, + correlation_id: CorrelationId::new(), + party_proofs: vec![ + PartyProofData { + party_id: 1, + address: honest, + proofs: vec![ + ( + ProofType::C1PkGeneration, + signals(0x42), + [0xa1; 32], + ArcBytes::from_bytes(&[0x01]), + ), + ( + ProofType::C2aSkShareComputation, + signals(0x42), + [0xa2; 32], + ArcBytes::from_bytes(&[0x02]), + ), + ], + }, + PartyProofData { + party_id: 2, + address: faulty, + proofs: vec![ + ( + ProofType::C1PkGeneration, + signals(0x11), + [0xb1; 32], + ArcBytes::from_bytes(&[0x03]), + ), + ( + ProofType::C2aSkShareComputation, + signals(0x99), + [0xb2; 32], + ArcBytes::from_bytes(&[0x04]), + ), + ], + }, + ], + }; + + let outcome = svc.on_check_requested(req).expect("same e3"); + assert!( + outcome.complete.inconsistent_parties.contains(&2), + "faulty party must be flagged" + ); + assert!( + !outcome.complete.inconsistent_parties.contains(&1), + "honest party must not be flagged" + ); + assert_eq!(outcome.violations.len(), 1); + assert_eq!(outcome.violations[0].accused_party_id, 2); + + // The faulty party's cache entries are evicted, so a later post-ZK + // event for the honest party does not re-report the faulty one. + let v = svc.on_proof_verified(passed( + e3(), + 1, + honest, + ProofType::C1PkGeneration, + [0xa1; 32], + signals(0x42), + )); + assert!(v.is_empty(), "evicted faulty party must not resurface"); +} + +#[test] +fn c2_sender_at_or_above_h_skips_c4_cross_check() { + let link = Box::new(TestLink { + scope: LinkScope::SourceMustExistInTargets, + source: ProofType::C2aSkShareComputation, + target: ProofType::C4aSkShareDecryption, + }); + let mut svc = CommitmentConsistency::new(e3(), vec![link], 2); + + // Party 2 (>= H) C2 cannot appear in C4 rows; must not be faulted. + svc.on_proof_verified(passed( + e3(), + 2, + addr(0x22), + ProofType::C2aSkShareComputation, + [0xc2; 32], + signals(0x22), + )); + let violations = svc.on_proof_verified(passed( + e3(), + 1, + addr(0x11), + ProofType::C4aSkShareDecryption, + [0xc4; 32], + signals(0x11), + )); + assert!( + violations.is_empty(), + "party_id >= H must be outside C4 expected_commitments roster" + ); +} diff --git a/crates/slashing/src/domain.rs b/crates/slashing/src/domain.rs new file mode 100644 index 0000000000..5359dbd06e --- /dev/null +++ b/crates/slashing/src/domain.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Compatibility view of pure fault-attribution modules stored by capability. + +#[path = "commitment_consistency/workflow.rs"] +pub(crate) mod commitment_consistency; diff --git a/crates/slashing/src/domain/accusation_voting.rs b/crates/slashing/src/domain/accusation_voting.rs deleted file mode 100644 index 88edec1156..0000000000 --- a/crates/slashing/src/domain/accusation_voting.rs +++ /dev/null @@ -1,1673 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Plain, synchronous domain service for the off-chain accusation quorum -//! protocol. -//! -//! This module contains **all** the business logic that used to live inside -//! the `AccusationManager` actix actor: -//! -//! - EIP-712 digest computation (accusation + vote) -//! - ECDSA signature creation and verification -//! - deadline stamping / peer-deadline validation -//! - pending-accusation bookkeeping (votes, dedup, buffering) -//! - vote tallying, quorum threshold checks, and equivocation detection -//! -//! [`AccusationVoting`] owns the protocol state and exposes plain methods that -//! mutate that state and **return a list of [`VoteAction`]s** describing the -//! I/O the actor must perform (publish a gossip event, dispatch a ZK request, -//! start/cancel a vote timeout). The service itself performs **no** I/O: it -//! never touches the event bus, the actix context, or timers. This makes the -//! whole protocol deterministically unit-testable without spinning up an actor -//! system. - -use std::collections::{HashMap, HashSet}; -use std::sync::Arc; -use std::time::Duration; - -use alloy::primitives::{keccak256, Address, Bytes, U256}; -use alloy::signers::local::PrivateKeySigner; -use alloy::signers::SignerSync; -use alloy::sol_types::SolValue; -use e3_events::{ - AccusationOutcome, AccusationQuorumReached, AccusationVote, CommitmentConsistencyViolation, - ComputeRequest, ComputeRequestError, ComputeResponse, ComputeResponseKind, CorrelationId, E3id, - EventContext, PartyProofsToVerify, ProofFailureAccusation, ProofType, ProofVerificationFailed, - ProofVerificationPassed, Sequenced, SignedProofPayload, SlashExecuted, TypedEvent, - VerifyShareProofsRequest, ZkRequest, ZkResponse, VOTE_DOMAIN_NAME, VOTE_DOMAIN_VERSION, - VOTE_TYPEHASH_STR, -}; -use e3_utils::ArcBytes; -use e3_zk_helpers::CiphernodesCommitteeSize; -use tracing::{error, info, warn}; - -use crate::actors::accusation_manager::Clock; - -/// How long to wait for votes before declaring the accusation inconclusive. -pub(crate) const DEFAULT_VOTE_TIMEOUT: Duration = Duration::from_secs(300); // 5 minutes - -/// An I/O effect the actor must perform on behalf of the [`AccusationVoting`] -/// service. The service returns these instead of performing the I/O itself, so -/// all protocol logic stays pure and testable. -pub(crate) enum VoteAction { - /// Broadcast our own accusation over gossip. `dedup_key` lets the actor - /// roll back the dedup entry if the (rare) publish fails, preserving the - /// original actor's behaviour of re-allowing the accusation on a dead bus. - PublishAccusation { - accusation: ProofFailureAccusation, - ec: EventContext, - dedup_key: (Address, ProofType), - }, - /// Broadcast an [`AccusationVote`] over gossip. - PublishVote { - vote: AccusationVote, - ec: EventContext, - }, - /// Publish the terminal [`AccusationQuorumReached`] decision. - PublishQuorum { - quorum: AccusationQuorumReached, - ec: EventContext, - }, - /// Dispatch an async ZK re-verification request (C3a/C3b forwarding). - /// `correlation_id` lets the actor discard the pending re-verification if - /// the publish fails. - DispatchZk { - request: ComputeRequest, - ec: EventContext, - correlation_id: CorrelationId, - }, - /// Start the vote-collection timeout for an accusation. - StartTimeout([u8; 32]), - /// Cancel a previously started vote-collection timeout (early quorum). - CancelTimeout([u8; 32]), -} - -/// An active accusation awaiting agreement votes from committee members. -/// -/// There is no `votes_against` field: a peer who finds the disputed proof -/// passes simply stays silent rather than broadcasting a signed disagreement. -/// The accusation runs to quorum or to `vote_timeout`. -pub(crate) struct PendingAccusation { - pub(crate) accusation: ProofFailureAccusation, - pub(crate) votes_for: Vec, - /// The EventContext from when this accusation was created — used for - /// timeout emission. - pub(crate) ec: EventContext, -} - -/// Cached verification result for a proof from a specific (accused, proof_type) -/// pair. Populated as proofs are received and verified (pass or fail). -struct ReceivedProofData { - data_hash: [u8; 32], - /// `true` if our local verification passed, `false` if it failed. - verification_passed: bool, - /// Raw `abi.encode(proof.data, proof.public_signals)` — preimage of - /// `data_hash`. Forwarded to the on-chain slashing contract so it can - /// recompute and verify the dataHash bound in voter signatures. - evidence: Bytes, -} - -/// Tracks an in-flight ZK re-verification for a forwarded C3a/C3b proof. -struct PendingReVerification { - accusation_id: [u8; 32], - data_hash: [u8; 32], - accused: Address, - proof_type: ProofType, - /// Evidence preimage bytes from the forwarded proof. - evidence: Bytes, -} - -/// Pure, synchronous core of the accusation quorum protocol. -/// -/// Owns all protocol state. Every public method mutates this state and returns -/// the I/O the actor must perform. It performs no I/O of its own. -pub(crate) struct AccusationVoting { - e3_id: E3id, - my_address: Address, - signer: PrivateKeySigner, - - /// On-chain `SlashingManager` address (EIP-712 `verifyingContract`). - slashing_manager: Address, - - /// All committee member addresses for this E3. - committee: Vec
, - /// Polynomial threshold `T`, used only for ZK circuit artifact resolution. - circuit_threshold_t: usize, - /// Vote quorum `H`, matching the on-chain `SlashingManager` threshold. - vote_quorum_h: usize, - /// Original committee N fixed at construction for ZK circuit resolution. - /// Do not derive from [`Self::committee`] after [`Self::on_slash_executed`] shrinks the roster. - committee_n: usize, - - /// Active accusations keyed by accusation_id. - pending: HashMap<[u8; 32], PendingAccusation>, - - /// Dedup: (accused, proof_type) pairs we've already accused. - accused_proofs: HashSet<(Address, ProofType)>, - - /// Cache of received data hashes per (accused, proof_type). - received_data: HashMap<(Address, ProofType), ReceivedProofData>, - - /// Votes received before the corresponding accusation. - buffered_votes: HashMap<[u8; 32], Vec>, - - /// In-flight C3a/C3b ZK re-verifications, keyed by CorrelationId. - pending_reverifications: HashMap, - - /// Vote timeout duration. - vote_timeout: Duration, - - /// Registry-wide off-chain freshness window (seconds). - vote_validity_secs: u64, - /// Clock-skew allowance when validating peer accusation deadlines. - accusation_deadline_skew_secs: u64, - - /// Wall-clock source used to derive accusation deadlines. - clock: Arc, - - /// BFV preset for circuit artifact resolution. - params_preset: e3_fhe_params::BfvPreset, -} - -impl AccusationVoting { - #[allow(clippy::too_many_arguments)] - pub(crate) fn new( - e3_id: E3id, - signer: PrivateKeySigner, - slashing_manager: Address, - committee: Vec
, - circuit_threshold_t: usize, - vote_quorum_h: usize, - vote_validity_secs: u64, - accusation_deadline_skew_secs: u64, - params_preset: e3_fhe_params::BfvPreset, - clock: Arc, - ) -> Self { - let my_address = signer.address(); - let committee_n = committee.len(); - Self { - e3_id, - my_address, - signer, - slashing_manager, - committee, - circuit_threshold_t, - vote_quorum_h, - committee_n, - pending: HashMap::new(), - accused_proofs: HashSet::new(), - received_data: HashMap::new(), - buffered_votes: HashMap::new(), - pending_reverifications: HashMap::new(), - vote_timeout: DEFAULT_VOTE_TIMEOUT, - vote_validity_secs, - accusation_deadline_skew_secs, - clock, - params_preset, - } - } - - /// The vote-collection timeout the actor should schedule. - pub(crate) fn vote_timeout(&self) -> Duration { - self.vote_timeout - } - - // ─── Deadline computation ──────────────────────────────────────────── - - /// Compute the on-chain vote-validity deadline (Unix seconds) the accuser - /// stamps on a fresh accusation. - fn compute_deadline(&self) -> u64 { - self.clock - .unix_now_secs() - .saturating_add(self.vote_validity_secs) - } - - /// Validate a peer-provided accusation deadline against this node's local - /// vote-validity policy and wall clock. - pub(crate) fn is_peer_deadline_acceptable( - deadline: u64, - now: u64, - vote_validity_secs: u64, - skew_secs: u64, - ) -> bool { - if vote_validity_secs == 0 { - return false; - } - let max_deadline = now - .saturating_add(vote_validity_secs) - .saturating_add(skew_secs); - deadline > now && deadline <= max_deadline - } - - // ─── Accusation ID computation ─────────────────────────────────────── - - /// Compute a deterministic ID for an accusation based on its key fields. - /// - /// `keccak256(abi.encodePacked(chainId, e3Id, accused, proofType))` - pub(crate) fn accusation_id(accusation: &ProofFailureAccusation) -> [u8; 32] { - let e3_id_u256: U256 = accusation - .e3_id - .clone() - .try_into() - .expect("E3id should be valid U256"); - let msg = ( - U256::from(accusation.e3_id.chain_id()), - e3_id_u256, - accusation.accused, - U256::from(accusation.proof_type as u8), - ) - .abi_encode_packed(); - keccak256(&msg).into() - } - - // ─── Signing / Verification ────────────────────────────────────────── - - fn sign_accusation_digest( - &self, - accusation: &ProofFailureAccusation, - ) -> Result, alloy::signers::Error> { - let digest = Self::accusation_digest(accusation); - let sig = self.signer.sign_message_sync(&digest)?; - Ok(sig.as_bytes().to_vec()) - } - - /// Structured digest for ECDSA signing of accusations. Off-chain only. - pub(crate) fn accusation_digest(accusation: &ProofFailureAccusation) -> [u8; 32] { - let e3_id_u256: U256 = accusation - .e3_id - .clone() - .try_into() - .expect("E3id should be valid U256"); - let typehash: [u8; 32] = keccak256( - "ProofFailureAccusation(uint256 chainId,uint256 e3Id,address accuser,address accused,uint256 proofType,bytes32 dataHash,uint256 deadline)" - ).into(); - let encoded = ( - typehash, - U256::from(accusation.e3_id.chain_id()), - e3_id_u256, - accusation.accuser, - accusation.accused, - U256::from(accusation.proof_type as u8), - accusation.data_hash, - U256::from(accusation.deadline), - ) - .abi_encode(); - keccak256(&encoded).into() - } - - fn verify_accusation_signature(&self, accusation: &ProofFailureAccusation) -> bool { - let digest = Self::accusation_digest(accusation); - let sig = match alloy::primitives::Signature::try_from( - accusation.signature.extract_bytes().as_ref(), - ) { - Ok(s) => s, - Err(_) => return false, - }; - match sig.recover_address_from_msg(digest) { - Ok(addr) => addr == accusation.accuser, - Err(_) => false, - } - } - - fn sign_vote_digest(&self, vote: &AccusationVote) -> Result, alloy::signers::Error> { - let digest = Self::vote_digest(vote, self.slashing_manager); - // `sign_hash_sync` signs the raw 32-byte hash without EIP-191 wrapping, - // which is what EIP-712 requires. - let sig = self.signer.sign_hash_sync(&digest.into())?; - Ok(sig.as_bytes().to_vec()) - } - - /// Canonical EIP-712 domain separator for vote signatures. - /// - /// Must match `SlashingManager`'s domain construction exactly. - fn vote_domain_separator(chain_id: u64, verifying_contract: Address) -> [u8; 32] { - let domain_typehash: [u8; 32] = keccak256( - "EIP712Domain(string name,string version,uint256 chainId,address verifyingContract)", - ) - .into(); - let name_hash: [u8; 32] = keccak256(VOTE_DOMAIN_NAME).into(); - let version_hash: [u8; 32] = keccak256(VOTE_DOMAIN_VERSION).into(); - let encoded = ( - domain_typehash, - name_hash, - version_hash, - U256::from(chain_id), - verifying_contract, - ) - .abi_encode(); - keccak256(&encoded).into() - } - - /// Canonical EIP-712 typed-data hash for a vote. - /// - /// `keccak256("\x19\x01" || domainSeparator || structHash)`. - pub(crate) fn vote_digest(vote: &AccusationVote, verifying_contract: Address) -> [u8; 32] { - let e3_id_u256: U256 = vote - .e3_id - .clone() - .try_into() - .expect("E3id should be valid U256"); - let typehash: [u8; 32] = keccak256(VOTE_TYPEHASH_STR).into(); - let struct_hash: [u8; 32] = keccak256( - ( - typehash, - e3_id_u256, - vote.accusation_id, - vote.voter, - vote.data_hash, - U256::from(vote.deadline), - ) - .abi_encode(), - ) - .into(); - let domain = Self::vote_domain_separator(vote.e3_id.chain_id(), verifying_contract); - let mut buf = Vec::with_capacity(2 + 32 + 32); - buf.push(0x19); - buf.push(0x01); - buf.extend_from_slice(&domain); - buf.extend_from_slice(&struct_hash); - keccak256(&buf).into() - } - - fn verify_vote_signature(&self, vote: &AccusationVote) -> bool { - let digest = Self::vote_digest(vote, self.slashing_manager); - let sig = - match alloy::primitives::Signature::try_from(vote.signature.extract_bytes().as_ref()) { - Ok(s) => s, - Err(_) => return false, - }; - match sig.recover_address_from_prehash(&digest.into()) { - Ok(addr) => addr == vote.voter, - Err(_) => false, - } - } - - /// Compute a keccak256 hash of a SignedProofPayload for data_hash comparison. - fn compute_payload_hash(payload: &SignedProofPayload) -> [u8; 32] { - let msg = ( - Bytes::copy_from_slice(&payload.payload.proof.data), - Bytes::copy_from_slice(&payload.payload.proof.public_signals), - ) - .abi_encode(); - keccak256(&msg).into() - } - - // ─── Caching ───────────────────────────────────────────────────────── - - /// Cache a successful (or failed) proof verification result. - pub(crate) fn cache_verification_result( - &mut self, - accused: Address, - proof_type: ProofType, - data_hash: [u8; 32], - passed: bool, - evidence: Bytes, - ) { - self.received_data.insert( - (accused, proof_type), - ReceivedProofData { - data_hash, - verification_passed: passed, - evidence, - }, - ); - } - - /// Cache a successful proof verification reported via `ProofVerificationPassed`. - pub(crate) fn on_proof_verification_passed(&mut self, data: ProofVerificationPassed) { - if data.e3_id != self.e3_id { - return; - } - if !self.committee.contains(&data.address) { - return; - } - // Evidence preimage = `abi.encode(proof.data, public_signals)`. - let evidence: Bytes = ( - Bytes::copy_from_slice(&data.proof_data), - Bytes::copy_from_slice(&data.public_signals), - ) - .abi_encode() - .into(); - self.received_data.insert( - (data.address, data.proof_type), - ReceivedProofData { - data_hash: data.data_hash, - verification_passed: true, - evidence, - }, - ); - } - - // ─── Rollback helpers (publish-failure paths) ──────────────────────── - - /// Roll back an initiation whose accusation broadcast failed. Mirrors the - /// original actor's behaviour of removing the dedup entry so a future - /// identical failure may retry. - pub(crate) fn rollback_initiation(&mut self, dedup_key: &(Address, ProofType)) { - self.accused_proofs.remove(dedup_key); - } - - /// Discard a pending ZK re-verification whose dispatch failed. - pub(crate) fn discard_reverification(&mut self, correlation_id: &CorrelationId) { - self.pending_reverifications.remove(correlation_id); - } - - // ─── Core Protocol ─────────────────────────────────────────────────── - - /// Called when the local node detects a proof failure. - pub(crate) fn on_local_proof_failure( - &mut self, - event: ProofVerificationFailed, - ec: &EventContext, - ) -> Vec { - if event.e3_id != self.e3_id { - return Vec::new(); - } - - let accused_address = if event.accused_address == Address::ZERO { - if let Some(&addr) = self.committee.get(event.accused_party_id as usize) { - warn!( - "Resolved Address::ZERO for party {} to committee address {}", - event.accused_party_id, addr - ); - addr - } else { - error!( - "Cannot resolve address for party {} (out of committee bounds) — dropping accusation", - event.accused_party_id - ); - return Vec::new(); - } - } else { - event.accused_address - }; - - if !self.committee.contains(&accused_address) { - warn!( - "Ignoring proof failure for {} — not on E3 {} committee", - accused_address, self.e3_id - ); - return Vec::new(); - } - - // Cache the failed verification result. - let evidence = Bytes::from( - ( - Bytes::copy_from_slice(&event.signed_payload.payload.proof.data), - Bytes::copy_from_slice(&event.signed_payload.payload.proof.public_signals), - ) - .abi_encode(), - ); - self.received_data.insert( - (accused_address, event.proof_type), - ReceivedProofData { - data_hash: event.data_hash, - verification_passed: false, - evidence, - }, - ); - - // For C3a/C3b, include the signed payload so other nodes can re-verify - let forwarded_payload = match event.proof_type { - ProofType::C3aSkShareEncryption | ProofType::C3bESmShareEncryption => { - Some(event.signed_payload.clone()) - } - _ => None, - }; - - let mut actions = Vec::new(); - self.initiate_accusation( - accused_address, - event.accused_party_id, - event.proof_type, - event.data_hash, - forwarded_payload, - ec, - &mut actions, - ); - actions - } - - /// Called when the `CommitmentConsistencyChecker` detects a cross-circuit - /// commitment mismatch for a party. - pub(crate) fn on_consistency_violation( - &mut self, - data: CommitmentConsistencyViolation, - ec: &EventContext, - ) -> Vec { - if data.e3_id != self.e3_id { - return Vec::new(); - } - - if !self.committee.contains(&data.accused_address) { - warn!( - "Ignoring commitment violation for {} — not on E3 {} committee", - data.accused_address, self.e3_id - ); - return Vec::new(); - } - - self.received_data.insert( - (data.accused_address, data.proof_type), - ReceivedProofData { - data_hash: data.data_hash, - verification_passed: false, - evidence: data.evidence.clone(), - }, - ); - - let mut actions = Vec::new(); - self.initiate_accusation( - data.accused_address, - data.accused_party_id, - data.proof_type, - data.data_hash, - None, - ec, - &mut actions, - ); - actions - } - - /// Shared accusation creation and broadcast logic. - #[allow(clippy::too_many_arguments)] - fn initiate_accusation( - &mut self, - accused_address: Address, - accused_party_id: u64, - proof_type: ProofType, - data_hash: [u8; 32], - forwarded_payload: Option, - ec: &EventContext, - actions: &mut Vec, - ) { - if !self.committee.contains(&accused_address) { - warn!( - "Refusing accusation against {} — not on E3 {} committee", - accused_address, self.e3_id - ); - return; - } - - let key = (accused_address, proof_type); - - // Dedup: don't create multiple accusations for the same (accused, proof_type) - if !self.accused_proofs.insert(key) { - info!( - "Already accused {:?} for {:?} — skipping duplicate", - accused_address, proof_type - ); - return; - } - - // Governance-disabled validity window means no accusation voting. - if self.vote_validity_secs == 0 { - warn!( - "Refusing accusation initiation for {:?} on E3 {}: vote_validity_secs is 0", - accused_address, self.e3_id - ); - self.accused_proofs.remove(&key); - return; - } - - // Pick the on-chain validity deadline once per accusation. - let deadline = self.compute_deadline(); - - // Create the accusation - let mut accusation = ProofFailureAccusation { - e3_id: self.e3_id.clone(), - accuser: self.my_address, - accused: accused_address, - accused_party_id, - proof_type, - data_hash, - deadline, - signed_payload: forwarded_payload, - signature: ArcBytes::default(), - }; - match self.sign_accusation_digest(&accusation) { - Ok(sig) => accusation.signature = ArcBytes::from_bytes(&sig), - Err(err) => { - error!("Failed to sign ProofFailureAccusation: {err}"); - self.accused_proofs.remove(&key); - return; - } - } - - let accusation_id = Self::accusation_id(&accusation); - - info!( - "Broadcasting accusation against {} for {:?} failure", - accused_address, proof_type - ); - - // Broadcast accusation via gossip - actions.push(VoteAction::PublishAccusation { - accusation: accusation.clone(), - ec: ec.clone(), - dedup_key: key, - }); - - // Cast our own agreement vote (we just observed the failure locally). - let mut own_vote = AccusationVote { - e3_id: self.e3_id.clone(), - accusation_id, - voter: self.my_address, - data_hash, - deadline, - signature: ArcBytes::default(), - }; - match self.sign_vote_digest(&own_vote) { - Ok(sig) => own_vote.signature = ArcBytes::from_bytes(&sig), - Err(err) => { - error!("Failed to sign own AccusationVote: {err}"); - self.accused_proofs.remove(&key); - return; - } - } - - actions.push(VoteAction::PublishVote { - vote: own_vote.clone(), - ec: ec.clone(), - }); - - // Start timeout - actions.push(VoteAction::StartTimeout(accusation_id)); - - // Store pending accusation with own vote - self.pending.insert( - accusation_id, - PendingAccusation { - accusation, - votes_for: vec![own_vote], - ec: ec.clone(), - }, - ); - - // Replay any votes that arrived before this accusation - if let Some(buffered) = self.buffered_votes.remove(&accusation_id) { - for vote in buffered { - self.on_vote_received_inner(vote, ec, actions); - } - } - - // Check quorum immediately (defensive for a future H=1 committee). - self.check_quorum(accusation_id, ec, actions); - } - - /// Called when we receive an accusation from another node via gossip. - pub(crate) fn on_accusation_received( - &mut self, - accusation: ProofFailureAccusation, - ec: &EventContext, - ) -> Vec { - let mut actions = Vec::new(); - self.on_accusation_received_inner(accusation, ec, &mut actions); - actions - } - - fn on_accusation_received_inner( - &mut self, - accusation: ProofFailureAccusation, - ec: &EventContext, - actions: &mut Vec, - ) { - // Ignore accusations for other E3s - if accusation.e3_id != self.e3_id { - return; - } - - let now = self.clock.unix_now_secs(); - if !Self::is_peer_deadline_acceptable( - accusation.deadline, - now, - self.vote_validity_secs, - self.accusation_deadline_skew_secs, - ) { - let max_deadline = now - .saturating_add(self.vote_validity_secs) - .saturating_add(self.accusation_deadline_skew_secs); - warn!( - "Ignoring accusation from {} — deadline {} outside local validity window \ - (now={}, vote_validity_secs={}, skew_secs={}, max_accepted_deadline={})", - accusation.accuser, - accusation.deadline, - now, - self.vote_validity_secs, - self.accusation_deadline_skew_secs, - max_deadline - ); - return; - } - - // Verify accuser is in committee - if !self.committee.contains(&accusation.accuser) { - warn!( - "Ignoring accusation from non-committee member {}", - accusation.accuser - ); - return; - } - - // Verify accused is a committee member (defense-in-depth) - if !self.committee.contains(&accusation.accused) { - warn!( - "Ignoring accusation against non-committee member {}", - accusation.accused - ); - return; - } - - // Ignore our own accusations (we already voted) - if accusation.accuser == self.my_address { - return; - } - - // Verify accuser's ECDSA signature - if !self.verify_accusation_signature(&accusation) { - warn!( - "Invalid signature on accusation from {} — ignoring", - accusation.accuser - ); - return; - } - - let accusation_id = Self::accusation_id(&accusation); - - // Don't process duplicate accusations - if self.pending.contains_key(&accusation_id) { - return; - } - - // Determine our position based on our local verification state. - let key = (accusation.accused, accusation.proof_type); - let our_data_hash = if let Some(received) = self.received_data.get(&key) { - if received.verification_passed { - info!( - "Local verification of {:?} from {} passed — abstaining \ - (no disagreement vote on the wire)", - accusation.proof_type, accusation.accused - ); - return; - } - received.data_hash - } else if let Some(ref forwarded) = accusation.signed_payload { - // C3a/C3b case: we didn't receive this proof directly. - let forwarded_valid = match forwarded.recover_address() { - Ok(addr) => { - if addr != accusation.accused { - warn!( - "Forwarded C3a/C3b payload signer {} != accused {} — cannot verify", - addr, accusation.accused - ); - false - } else if forwarded.payload.e3_id != self.e3_id { - warn!("Forwarded C3a/C3b payload e3_id mismatch — cannot verify"); - false - } else { - let expected = forwarded.payload.proof_type.circuit_names(); - expected.contains(&forwarded.payload.proof.circuit) - } - } - Err(e) => { - warn!("Forwarded C3a/C3b payload signature invalid: {e} — cannot verify"); - false - } - }; - - if !forwarded_valid { - // Can't trust the forwarded proof — abstain - return; - } - - // Bind the forwarded proof to the accusation. - if forwarded.payload.proof_type != accusation.proof_type { - warn!( - "Forwarded C3a/C3b proof_type {:?} != accusation proof_type {:?} — cannot verify", - forwarded.payload.proof_type, accusation.proof_type - ); - return; - } - let computed_hash = Self::compute_payload_hash(forwarded); - if computed_hash != accusation.data_hash { - warn!( - "Forwarded C3a/C3b data_hash mismatch (len {} vs {}) — cannot verify", - computed_hash.len(), - accusation.data_hash.len() - ); - return; - } - - let data_hash = Self::compute_payload_hash(forwarded); - let evidence: Bytes = ( - Bytes::copy_from_slice(&forwarded.payload.proof.data), - Bytes::copy_from_slice(&forwarded.payload.proof.public_signals), - ) - .abi_encode() - .into(); - let accused_party_id = accusation.accused_party_id; - let forwarded_clone = forwarded.clone(); - - let committee_size = match CiphernodesCommitteeSize::from_threshold( - self.circuit_threshold_t, - self.committee_n, - ) { - Ok(c) => c, - Err(e) => { - warn!("Cannot derive committee size for ZK re-verification: {e}"); - return; - } - }; - - // Create PendingAccusation without our vote — it arrives after ZK completes. - actions.push(VoteAction::StartTimeout(accusation_id)); - self.pending.insert( - accusation_id, - PendingAccusation { - accusation, - votes_for: Vec::new(), - ec: ec.clone(), - }, - ); - - // Replay any buffered votes - if let Some(buffered) = self.buffered_votes.remove(&accusation_id) { - for vote in buffered { - self.on_vote_received_inner(vote, ec, actions); - } - } - - // Dispatch ZK re-verification - let correlation_id = CorrelationId::new(); - self.pending_reverifications.insert( - correlation_id, - PendingReVerification { - accusation_id, - data_hash, - accused: key.0, - proof_type: key.1, - evidence, - }, - ); - - let party_proof = PartyProofsToVerify { - sender_party_id: accused_party_id, - signed_proofs: vec![forwarded_clone], - }; - let request = ComputeRequest::zk( - ZkRequest::VerifyShareProofs(VerifyShareProofsRequest { - party_proofs: vec![party_proof], - params_preset: self.params_preset, - committee_size, - }), - correlation_id, - self.e3_id.clone(), - ); - - actions.push(VoteAction::DispatchZk { - request, - ec: ec.clone(), - correlation_id, - }); - - // Vote deferred — return without falling through to the normal vote path - return; - } else { - // We don't have the data and no payload was forwarded — abstain - info!( - "No local data for accused {} proof {:?} — abstaining from vote", - accusation.accused, accusation.proof_type - ); - return; - }; - - // We saw the proof fail locally — agree with the accusation. - let mut vote = AccusationVote { - e3_id: self.e3_id.clone(), - accusation_id, - voter: self.my_address, - data_hash: our_data_hash, - deadline: accusation.deadline, - signature: ArcBytes::default(), - }; - match self.sign_vote_digest(&vote) { - Ok(sig) => vote.signature = ArcBytes::from_bytes(&sig), - Err(err) => { - error!("Failed to sign AccusationVote: {err}"); - return; - } - } - - info!( - "Agreeing with accusation against {} for {:?}", - accusation.accused, accusation.proof_type - ); - - // Broadcast vote via gossip - actions.push(VoteAction::PublishVote { - vote: vote.clone(), - ec: ec.clone(), - }); - - // Start timeout for this accusation - actions.push(VoteAction::StartTimeout(accusation_id)); - - // Record in pending - let pending = PendingAccusation { - accusation, - votes_for: vec![vote], - ec: ec.clone(), - }; - self.pending.insert(accusation_id, pending); - - // Replay any votes that arrived before this accusation - if let Some(buffered) = self.buffered_votes.remove(&accusation_id) { - for vote in buffered { - self.on_vote_received_inner(vote, ec, actions); - } - } - - // Check quorum - self.check_quorum(accusation_id, ec, actions); - } - - /// Called when we receive a vote from another node via gossip. - pub(crate) fn on_vote_received( - &mut self, - vote: AccusationVote, - ec: &EventContext, - ) -> Vec { - let mut actions = Vec::new(); - self.on_vote_received_inner(vote, ec, &mut actions); - actions - } - - fn on_vote_received_inner( - &mut self, - vote: AccusationVote, - ec: &EventContext, - actions: &mut Vec, - ) { - // Ignore votes for other E3s - if vote.e3_id != self.e3_id { - return; - } - - // Verify voter is in committee - if !self.committee.contains(&vote.voter) { - warn!("Ignoring vote from non-committee member {}", vote.voter); - return; - } - - // Ignore our own votes (already recorded) - if vote.voter == self.my_address { - return; - } - - // Verify voter's ECDSA signature - if !self.verify_vote_signature(&vote) { - warn!("Invalid signature on vote from {} — ignoring", vote.voter); - return; - } - - let vote_accusation_id = vote.accusation_id; - - // Find the pending accusation - let Some(pending) = self.pending.get_mut(&vote_accusation_id) else { - // Unknown accusation — buffer the vote for replay. - let committee_len = self.committee.len(); - let buf = self.buffered_votes.entry(vote_accusation_id).or_default(); - if buf.len() < committee_len { - buf.push(vote); - } else { - warn!( - "Buffered votes for unknown accusation {:?} reached committee-size cap — dropping vote", - vote_accusation_id - ); - } - return; - }; - - // Reject votes whose deadline disagrees with the accusation's deadline. - if vote.deadline != pending.accusation.deadline { - warn!( - "Ignoring vote from {} — deadline {} does not match accusation deadline {}", - vote.voter, vote.deadline, pending.accusation.deadline - ); - return; - } - - // Reject votes from the accused party — conflict of interest - if vote.voter == pending.accusation.accused { - warn!( - "Ignoring vote from accused party {} on their own accusation", - vote.voter - ); - return; - } - - // Dedup: don't count same voter twice - let already_voted = pending.votes_for.iter().any(|v| v.voter == vote.voter); - if already_voted { - return; - } - - // Accuser's vote data_hash must match the accusation's data_hash. - if vote.voter == pending.accusation.accuser - && vote.data_hash != pending.accusation.data_hash - { - warn!( - "Accuser {} sent vote with data_hash inconsistent with their accusation — rejecting vote", - vote.voter - ); - return; - } - - // Every received `AccusationVote` is an agreement. - pending.votes_for.push(vote); - - self.check_quorum(vote_accusation_id, ec, actions); - } - - /// Evaluate whether we have enough agreeing votes to decide. - fn check_quorum( - &mut self, - accusation_id: [u8; 32], - ec: &EventContext, - actions: &mut Vec, - ) { - let Some(pending) = self.pending.get(&accusation_id) else { - return; - }; - - let agree_count = pending.votes_for.len(); - if agree_count < self.vote_quorum_h { - // Not yet at quorum. - return; - } - - // Reached `H` — decide between AccusedFaulted and Equivocation. - let agree_hashes: HashSet<[u8; 32]> = - pending.votes_for.iter().map(|v| v.data_hash).collect(); - if agree_hashes.len() > 1 { - info!( - "Equivocation detected at quorum: {} unique data hashes among {} agreeing voters for {} {:?}", - agree_hashes.len(), - agree_count, - pending.accusation.accused, - pending.accusation.proof_type - ); - self.emit_quorum_reached(accusation_id, AccusationOutcome::Equivocation, ec, actions); - } else { - info!( - "Quorum reached: {} votes confirm {} sent bad {:?} proof — AccusedFaulted", - agree_count, pending.accusation.accused, pending.accusation.proof_type - ); - self.emit_quorum_reached( - accusation_id, - AccusationOutcome::AccusedFaulted, - ec, - actions, - ); - } - } - - /// Called when the vote timeout expires for an accusation. Returns the - /// terminal quorum event the actor must publish, if the accusation was - /// still pending. - pub(crate) fn on_vote_timeout( - &mut self, - accusation_id: [u8; 32], - ) -> Option<(AccusationQuorumReached, EventContext)> { - let pending = self.pending.remove(&accusation_id)?; // Already resolved - - let outcome = if pending.votes_for.len() >= self.vote_quorum_h { - let agree_hashes: HashSet<[u8; 32]> = - pending.votes_for.iter().map(|v| v.data_hash).collect(); - if agree_hashes.len() > 1 { - AccusationOutcome::Equivocation - } else { - AccusationOutcome::AccusedFaulted - } - } else { - AccusationOutcome::Inconclusive - }; - - warn!( - "Accusation against {} for {:?} timed out with {} agreeing votes — outcome: {:?}", - pending.accusation.accused, - pending.accusation.proof_type, - pending.votes_for.len(), - outcome - ); - - let evidence = self - .received_data - .get(&(pending.accusation.accused, pending.accusation.proof_type)) - .map(|d| d.evidence.clone()) - .unwrap_or_default(); - Some(( - AccusationQuorumReached { - e3_id: self.e3_id.clone(), - accuser: pending.accusation.accuser, - accused: pending.accusation.accused, - proof_type: pending.accusation.proof_type, - votes_for: pending.votes_for, - outcome, - evidence, - }, - pending.ec, - )) - } - - fn emit_quorum_reached( - &mut self, - accusation_id: [u8; 32], - outcome: AccusationOutcome, - ec: &EventContext, - actions: &mut Vec, - ) { - let Some(pending) = self.pending.remove(&accusation_id) else { - return; - }; - - // Cancel the timeout to avoid unnecessary timer fires - actions.push(VoteAction::CancelTimeout(accusation_id)); - - info!( - "Accusation quorum reached for {} {:?}: {} agreeing votes — outcome: {}", - pending.accusation.accused, - pending.accusation.proof_type, - pending.votes_for.len(), - outcome - ); - - let evidence = self - .received_data - .get(&(pending.accusation.accused, pending.accusation.proof_type)) - .map(|d| d.evidence.clone()) - .unwrap_or_default(); - actions.push(VoteAction::PublishQuorum { - quorum: AccusationQuorumReached { - e3_id: self.e3_id.clone(), - accuser: pending.accusation.accuser, - accused: pending.accusation.accused, - proof_type: pending.accusation.proof_type, - votes_for: pending.votes_for, - outcome, - evidence, - }, - ec: ec.clone(), - }); - } - - /// Handle an on-chain SlashExecuted event for this E3. - pub(crate) fn on_slash_executed(&mut self, data: SlashExecuted) { - if data.e3_id != self.e3_id { - return; - } - let prev_len = self.committee.len(); - self.committee.retain(|addr| *addr != data.operator); - if self.committee.len() < prev_len { - info!( - "Removed slashed operator {} from committee (now {} members)", - data.operator, - self.committee.len() - ); - - // Purge any votes from the expelled node in pending accusations - for pending in self.pending.values_mut() { - pending.votes_for.retain(|v| v.voter != data.operator); - } - - // Purge from buffered votes - for buf in self.buffered_votes.values_mut() { - buf.retain(|v| v.voter != data.operator); - } - } - } - - /// Handle ZK re-verification response for a forwarded C3a/C3b proof. - pub(crate) fn handle_reverification_response( - &mut self, - msg: TypedEvent, - ) -> Vec { - let (msg, _ec) = msg.into_components(); - let mut actions = Vec::new(); - - let correlation_id = msg.correlation_id; - let Some(reverif) = self.pending_reverifications.remove(&correlation_id) else { - return actions; // Not our correlation ID - }; - - let zk_passed = match msg.response { - ComputeResponseKind::Zk(ZkResponse::VerifyShareProofs(r)) => { - if r.party_results.is_empty() { - warn!("Empty ZK re-verification results — abstaining"); - return actions; - } - r.party_results.first().is_some_and(|r| r.all_verified) - } - _ => { - warn!("Unexpected ComputeResponse kind for C3a/C3b re-verification — abstaining"); - return actions; - } - }; - - // Cache the result for future accusations regardless of outcome. - self.cache_verification_result( - reverif.accused, - reverif.proof_type, - reverif.data_hash, - zk_passed, - reverif.evidence.clone(), - ); - - // ZK re-verification passed ⇒ proof is valid ⇒ we disagree ⇒ abstain. - if zk_passed { - info!( - "C3a/C3b re-verification passed for {:?} — abstaining from vote", - reverif.proof_type - ); - return actions; - } - - // ZK re-verification failed ⇒ we agree with the accusation. - let (ec, deadline) = match self.pending.get(&reverif.accusation_id) { - Some(pending) => (pending.ec.clone(), pending.accusation.deadline), - None => { - // Accusation already resolved before ZK finished - return actions; - } - }; - - let mut vote = AccusationVote { - e3_id: self.e3_id.clone(), - accusation_id: reverif.accusation_id, - voter: self.my_address, - data_hash: reverif.data_hash, - deadline, - signature: ArcBytes::default(), - }; - match self.sign_vote_digest(&vote) { - Ok(sig) => vote.signature = ArcBytes::from_bytes(&sig), - Err(err) => { - error!("Failed to sign C3a/C3b AccusationVote: {err}"); - return actions; - } - } - - info!( - "C3a/C3b re-verification confirmed failure for {:?} — agreeing with accusation", - reverif.proof_type - ); - - // Broadcast vote via gossip - actions.push(VoteAction::PublishVote { - vote: vote.clone(), - ec: ec.clone(), - }); - - // Record in pending - if let Some(pending) = self.pending.get_mut(&reverif.accusation_id) { - pending.votes_for.push(vote); - } - - // Check quorum - self.check_quorum(reverif.accusation_id, &ec, &mut actions); - actions - } - - /// Handle ZK re-verification error for a forwarded C3a/C3b proof. - pub(crate) fn handle_reverification_error(&mut self, msg: TypedEvent) { - let (msg, _ec) = msg.into_components(); - - let correlation_id = msg.correlation_id(); - let Some(reverif) = self.pending_reverifications.remove(correlation_id) else { - return; // Not our correlation ID - }; - - error!( - "C3a/C3b ZK re-verification failed for {:?} — abstaining from vote", - reverif.proof_type - ); - // Don't vote — effectively abstain - } -} - -// ════════════════════════════════════════════════════════════════════════════ -// Tests -// ════════════════════════════════════════════════════════════════════════════ - -#[cfg(test)] -mod tests { - use super::*; - use alloy::primitives::FixedBytes; - use e3_events::{InterfoldEventData, Unsequenced}; - - struct FixedClock(u64); - impl Clock for FixedClock { - fn unix_now_secs(&self) -> u64 { - self.0 - } - } - - /// A throwaway sequenced [`EventContext`] for driving service calls in - /// tests. The service never inspects the context beyond cloning it onto - /// emitted actions, so any well-formed origin context works. - fn ctx() -> EventContext { - let vote = AccusationVote { - e3_id: E3id::new("42", CHAIN_ID), - accusation_id: [0u8; 32], - voter: Address::ZERO, - data_hash: [0u8; 32], - deadline: 0, - signature: ArcBytes::default(), - }; - EventContext::::from(InterfoldEventData::from(vote)).sequence(0) - } - - const CHAIN_ID: u64 = 31337; - const VALIDITY: u64 = 1_800; - const SKEW: u64 = 30; - const NOW: u64 = 1_700_000_000; - - fn signer(byte: u8) -> PrivateKeySigner { - let mut bytes = [0u8; 32]; - bytes[31] = byte; - PrivateKeySigner::from_bytes(&FixedBytes::<32>::from(bytes)).unwrap() - } - - fn voting_with( - me: &PrivateKeySigner, - committee: Vec
, - circuit_threshold_t: usize, - vote_quorum_h: usize, - ) -> AccusationVoting { - AccusationVoting::new( - E3id::new("42", CHAIN_ID), - me.clone(), - "0x9999999999999999999999999999999999999999" - .parse() - .unwrap(), - committee, - circuit_threshold_t, - vote_quorum_h, - VALIDITY, - SKEW, - e3_fhe_params::BfvPreset::default(), - Arc::new(FixedClock(NOW)), - ) - } - - /// Build and sign a vote as `who` for the given accusation/data hash. - fn signed_vote( - who: &PrivateKeySigner, - slashing_manager: Address, - e3_id: &E3id, - accusation_id: [u8; 32], - data_hash: [u8; 32], - deadline: u64, - ) -> AccusationVote { - let mut vote = AccusationVote { - e3_id: e3_id.clone(), - accusation_id, - voter: who.address(), - data_hash, - deadline, - signature: ArcBytes::default(), - }; - let digest = AccusationVoting::vote_digest(&vote, slashing_manager); - let sig = who.sign_hash_sync(&FixedBytes::<32>::from(digest)).unwrap(); - vote.signature = ArcBytes::from_bytes(&sig.as_bytes()); - vote - } - - fn insert_pending( - v: &mut AccusationVoting, - accuser: &PrivateKeySigner, - accused: Address, - data_hash: [u8; 32], - deadline: u64, - own_vote: AccusationVote, - ) -> [u8; 32] { - let accusation = ProofFailureAccusation { - e3_id: v.e3_id.clone(), - accuser: accuser.address(), - accused, - accused_party_id: 1, - proof_type: ProofType::C1PkGeneration, - data_hash, - deadline, - signed_payload: None, - signature: ArcBytes::default(), - }; - let id = AccusationVoting::accusation_id(&accusation); - v.pending.insert( - id, - PendingAccusation { - accusation, - votes_for: vec![own_vote], - ec: ctx(), - }, - ); - id - } - - /// Digest computation must be deterministic for identical inputs and must - /// differ when any bound field changes. - #[test] - fn vote_digest_is_deterministic() { - let sm: Address = "0x5555555555555555555555555555555555555555" - .parse() - .unwrap(); - let voter: Address = "0x2222222222222222222222222222222222222222" - .parse() - .unwrap(); - let vote = AccusationVote { - e3_id: E3id::new("42", CHAIN_ID), - accusation_id: [0xab; 32], - voter, - data_hash: [0xcd; 32], - deadline: NOW, - signature: ArcBytes::default(), - }; - let a = AccusationVoting::vote_digest(&vote, sm); - let b = AccusationVoting::vote_digest(&vote, sm); - assert_eq!(a, b, "vote digest must be deterministic"); - - let mut vote2 = vote.clone(); - vote2.deadline = NOW + 1; - assert_ne!( - a, - AccusationVoting::vote_digest(&vote2, sm), - "changing deadline must change the digest" - ); - } - - /// A second agreeing vote that reaches `vote_quorum_h` must produce a single - /// AccusedFaulted quorum decision and remove the pending accusation. - #[test] - fn tally_reaches_quorum_at_threshold() { - let me = signer(1); - let b = signer(2); - let accused = signer(9).address(); - let committee = vec![me.address(), b.address(), accused]; - let mut v = voting_with(&me, committee, 1, 2); - let sm = v.slashing_manager; - let data_hash = [0x11; 32]; - - let own = signed_vote(&me, sm, &v.e3_id, [0u8; 32], data_hash, NOW + VALIDITY); - let id = insert_pending(&mut v, &me, accused, data_hash, NOW + VALIDITY, own); - // own vote's accusation_id was a placeholder; fix it to the real id. - v.pending.get_mut(&id).unwrap().votes_for[0].accusation_id = id; - - let vote_b = signed_vote(&b, sm, &v.e3_id, id, data_hash, NOW + VALIDITY); - let actions = v.on_vote_received(vote_b, &ctx()); - - let quorum = actions - .iter() - .filter_map(|a| match a { - VoteAction::PublishQuorum { quorum, .. } => Some(quorum), - _ => None, - }) - .count(); - assert_eq!(quorum, 1, "exactly one quorum decision expected"); - assert!( - !v.pending.contains_key(&id), - "pending accusation removed after quorum" - ); - } - - /// Inserting the same voter twice must not double-count nor re-trigger quorum. - #[test] - fn idempotent_vote_insert() { - let me = signer(1); - let b = signer(2); - let accused = signer(9).address(); - let committee = vec![me.address(), b.address(), accused]; - let mut v = voting_with(&me, committee, 1, 3); // quorum above what 2 votes reach - let sm = v.slashing_manager; - let data_hash = [0x11; 32]; - - let own = signed_vote(&me, sm, &v.e3_id, [0u8; 32], data_hash, NOW + VALIDITY); - let id = insert_pending(&mut v, &me, accused, data_hash, NOW + VALIDITY, own); - v.pending.get_mut(&id).unwrap().votes_for[0].accusation_id = id; - - let vote_b = signed_vote(&b, sm, &v.e3_id, id, data_hash, NOW + VALIDITY); - let _ = v.on_vote_received(vote_b.clone(), &ctx()); - let len_after_first = v.pending.get(&id).unwrap().votes_for.len(); - - // Same voter again — must be ignored. - let actions = v.on_vote_received(vote_b, &ctx()); - let len_after_second = v.pending.get(&id).unwrap().votes_for.len(); - assert_eq!( - len_after_first, len_after_second, - "duplicate voter must not be counted twice" - ); - assert!( - actions.is_empty(), - "duplicate vote must not emit any actions" - ); - } - - /// Quorum must trigger exactly at the M-th agreeing vote, not before. - #[test] - fn quorum_boundary() { - let me = signer(1); - let b = signer(2); - let c = signer(3); - let accused = signer(9).address(); - let committee = vec![me.address(), b.address(), c.address(), accused]; - let mut v = voting_with(&me, committee, 1, 3); - let sm = v.slashing_manager; - let data_hash = [0x11; 32]; - - let own = signed_vote(&me, sm, &v.e3_id, [0u8; 32], data_hash, NOW + VALIDITY); - let id = insert_pending(&mut v, &me, accused, data_hash, NOW + VALIDITY, own); - v.pending.get_mut(&id).unwrap().votes_for[0].accusation_id = id; - - // 2nd vote — below threshold of 3, no quorum. - let vote_b = signed_vote(&b, sm, &v.e3_id, id, data_hash, NOW + VALIDITY); - let actions = v.on_vote_received(vote_b, &ctx()); - assert!( - actions.is_empty(), - "no quorum before reaching threshold M=3" - ); - assert!(v.pending.contains_key(&id)); - - // 3rd vote — reaches threshold, quorum fires. - let vote_c = signed_vote(&c, sm, &v.e3_id, id, data_hash, NOW + VALIDITY); - let actions = v.on_vote_received(vote_c, &ctx()); - assert!( - actions - .iter() - .any(|a| matches!(a, VoteAction::PublishQuorum { .. })), - "quorum must fire at the M-th vote" - ); - } - - /// Agreeing votes that disagree on data_hash at quorum yield Equivocation. - #[test] - fn equivocation_when_hashes_differ_at_quorum() { - let me = signer(1); - let b = signer(2); - let accused = signer(9).address(); - let committee = vec![me.address(), b.address(), accused]; - let mut v = voting_with(&me, committee, 1, 2); - let sm = v.slashing_manager; - let data_hash_a = [0x11; 32]; - let data_hash_b = [0x22; 32]; - - let own = signed_vote(&me, sm, &v.e3_id, [0u8; 32], data_hash_a, NOW + VALIDITY); - let id = insert_pending(&mut v, &me, accused, data_hash_a, NOW + VALIDITY, own); - v.pending.get_mut(&id).unwrap().votes_for[0].accusation_id = id; - - // Voter b agrees but reports a different data_hash → equivocation. - let vote_b = signed_vote(&b, sm, &v.e3_id, id, data_hash_b, NOW + VALIDITY); - let actions = v.on_vote_received(vote_b, &ctx()); - let outcome = actions.iter().find_map(|a| match a { - VoteAction::PublishQuorum { quorum, .. } => Some(quorum.outcome.clone()), - _ => None, - }); - assert_eq!( - outcome, - Some(AccusationOutcome::Equivocation), - "differing data hashes at quorum must yield Equivocation" - ); - } - - /// Timeout below threshold yields Inconclusive; at/above threshold yields - /// AccusedFaulted. - #[test] - fn timeout_outcome_depends_on_vote_count() { - let me = signer(1); - let accused = signer(9).address(); - let committee = vec![me.address(), signer(2).address(), accused]; - let mut v = voting_with(&me, committee, 1, 2); - let sm = v.slashing_manager; - let data_hash = [0x11; 32]; - - let own = signed_vote(&me, sm, &v.e3_id, [0u8; 32], data_hash, NOW + VALIDITY); - let id = insert_pending(&mut v, &me, accused, data_hash, NOW + VALIDITY, own); - - // Only one agreeing vote, threshold is 2 → Inconclusive. - let (quorum, _ec) = v.on_vote_timeout(id).expect("timeout emits a decision"); - assert_eq!(quorum.outcome, AccusationOutcome::Inconclusive); - assert!(v.on_vote_timeout(id).is_none(), "second timeout is a no-op"); - } - - /// After a slash shrinks the live roster, ZK re-verification must still use the - /// canonical circuit committee size cached at construction. - #[test] - fn committee_size_unchanged_after_slash() { - let me = signer(1); - // Micro committee (T=4, N=9) — canonical pair in `from_threshold`. - let committee: Vec
= (1..=9u8).map(|b| signer(b).address()).collect(); - let mut v = voting_with(&me, committee.clone(), 4, 5); - - let slashed = committee[8]; - v.on_slash_executed(SlashExecuted { - e3_id: v.e3_id.clone(), - proposal_id: 1, - operator: slashed, - reason: [0u8; 32], - ticket_amount: 0, - license_amount: 0, - }); - assert_eq!(v.committee.len(), 8); - assert_eq!(v.committee_n, 9); - // Shrunken roster must not drive circuit resolution. - assert!( - CiphernodesCommitteeSize::from_threshold(v.circuit_threshold_t, v.committee.len()) - .is_err() - ); - assert_eq!( - CiphernodesCommitteeSize::from_threshold(v.circuit_threshold_t, v.committee_n).unwrap(), - CiphernodesCommitteeSize::Micro - ); - } -} diff --git a/crates/slashing/src/domain/mod.rs b/crates/slashing/src/domain/mod.rs deleted file mode 100644 index 9944b22d4d..0000000000 --- a/crates/slashing/src/domain/mod.rs +++ /dev/null @@ -1,13 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Plain, synchronous domain services. These own all protocol state and -//! business logic; they perform **no** I/O (no event bus, no actix context, -//! no timers). The thin actors in [`crate::actors`] drive them and execute the -//! decisions/data they return. - -pub(crate) mod accusation_voting; -pub(crate) mod commitment_consistency; diff --git a/crates/slashing/src/lib.rs b/crates/slashing/src/lib.rs index 8bf295d673..aee83a6c6a 100644 --- a/crates/slashing/src/lib.rs +++ b/crates/slashing/src/lib.rs @@ -11,6 +11,7 @@ mod actors; mod domain; +mod workflow; pub mod accusation_manager_ext; pub mod commitment_consistency_checker_ext; diff --git a/crates/slashing/src/workflow.rs b/crates/slashing/src/workflow.rs new file mode 100644 index 0000000000..eff69110c2 --- /dev/null +++ b/crates/slashing/src/workflow.rs @@ -0,0 +1,10 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Compatibility view of fault-attribution workflows stored by capability. + +#[path = "accusation_voting/workflow.rs"] +pub(crate) mod accusation_voting; diff --git a/crates/sortition/src/actors.rs b/crates/sortition/src/actors.rs new file mode 100644 index 0000000000..8a09803114 --- /dev/null +++ b/crates/sortition/src/actors.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Compatibility view of actors stored with their sortition capabilities. + +#[path = "ciphernode_selection/actor.rs"] +mod ciphernode_selector; +#[path = "sortition/actor.rs"] +mod sortition; + +pub use ciphernode_selector::*; +pub use sortition::*; diff --git a/crates/sortition/src/actors/sortition.rs b/crates/sortition/src/actors/sortition.rs deleted file mode 100644 index bdb0972300..0000000000 --- a/crates/sortition/src/actors/sortition.rs +++ /dev/null @@ -1,689 +0,0 @@ -// SPDX-License-Identifier: LGPL-4.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use crate::domain::backends::{SortitionBackend, SortitionList}; -use crate::domain::node_registry::{NodeRegistry, NodeStateStore}; -use crate::domain::ticket_sortition; -use crate::messages::{ - CommitteeMembersResponse, E3CommitteeContainsRequest, E3CommitteeContainsResponse, - GetCommitteeMembersRequest, WithSortitionTicket, -}; -use crate::CiphernodeSelector; -use crate::FinalizedCommitteeRetention; -use actix::prelude::*; -use anyhow::{anyhow, Result}; -use e3_data::{AutoPersist, Persistable, Repository}; -use e3_events::{ - prelude::*, trap, CiphernodeAdded, CiphernodeRemoved, Committee, CommitteeFinalized, - CommitteeMemberExpelled, CommitteePublished, ConfigurationUpdated, E3Failed, E3RequestComplete, - E3Requested, E3Stage, E3StageChanged, EType, EventContext, EventType, InterfoldEvent, - OperatorActivationChanged, PlaintextOutputPublished, Seed, Sequenced, TicketBalanceUpdated, - TypedEvent, -}; -use e3_events::{BusHandle, E3id, InterfoldEventData}; -use e3_utils::{NotifySync, MAILBOX_LIMIT}; -use std::collections::HashMap; -use tracing::{info, instrument, warn}; - -/// Sortition actor that manages the sortition algorithm and the node state. -pub struct Sortition { - /// Persistent map of `chain_id -> SortitionBackend`. - backends: Persistable>, - /// Persistent map of `chain_id -> NodeStateStore`. - node_state: Persistable>, - /// Event bus for error reporting and interfold event subscription. - bus: BusHandle, - /// Persistent map of finalized committees per E3 - finalized_committees: Persistable>, - /// Address for the CiphernodeSelector - ciphernode_selector: Addr, - /// Address for the current node - address: String, - /// Ephemeral buffer of raw `CommitteeMemberExpelled` events that arrived before the matching - /// committee was finalized (e.g. out-of-order live delivery or a reorg). Drained when the - /// `CommitteeFinalized` event for the same E3 is processed so early expulsions are not lost. - pending_expulsions: HashMap)>>, -} - -/// Parameters for constructing a `Sortition` actor. -#[derive(Debug)] -pub struct SortitionParams { - /// Event bus address. - pub bus: BusHandle, - /// Persisted per-chain backend map. - pub backends: Persistable>, - /// Node state store per chain - pub node_state: Persistable>, - /// Persistent map of finalized committees per E3 - pub finalized_committees: Persistable>, - /// Address for the CiphernodeSelector - pub ciphernode_selector: Addr, - /// Address for the current node - pub address: String, -} - -impl Sortition { - pub fn new(params: SortitionParams) -> Self { - Self { - backends: params.backends, - node_state: params.node_state, - bus: params.bus, - finalized_committees: params.finalized_committees, - ciphernode_selector: params.ciphernode_selector, - address: params.address, - pending_expulsions: HashMap::new(), - } - } - - #[instrument(name = "sortition_attach", skip_all)] - pub async fn attach( - bus: &BusHandle, - backends_store: Repository>, - node_state_store: Repository>, - committees_store: Repository>, - default_backend: SortitionBackend, - ciphernode_selector: Addr, - address: &str, - ) -> Result> { - let mut backends = backends_store.load_or_default(HashMap::new()).await?; - let node_state = node_state_store.load_or_default(HashMap::new()).await?; - let finalized_committees = committees_store.load_or_default(HashMap::new()).await?; - - backends.try_mutate_without_context(|mut list| { - list.insert(u64::MAX, default_backend); - Ok(list) - })?; - - let addr = Sortition::new(SortitionParams { - bus: bus.clone(), - backends, - node_state, - finalized_committees, - ciphernode_selector, - address: address.to_owned(), - }) - .start(); - - // Subscribe to state-building events immediately (needed during EventStore replay) - bus.subscribe_all( - &[ - EventType::CiphernodeAdded, - EventType::CiphernodeRemoved, - EventType::TicketBalanceUpdated, - EventType::OperatorActivationChanged, - EventType::ConfigurationUpdated, - EventType::CommitteePublished, - EventType::PlaintextOutputPublished, - EventType::CommitteeFinalized, - EventType::CommitteeMemberExpelled, - EventType::E3Failed, - EventType::E3StageChanged, - EventType::E3RequestComplete, - ], - addr.clone().into(), - ); - - // Gate E3Requested behind EffectsEnabled — sortition should not trigger - // ticket generation during historical event replay. - bus.subscribe( - EventType::EffectsEnabled, - e3_events::run_once::({ - let bus = bus.clone(); - let addr = addr.clone(); - move |_| { - bus.subscribe(EventType::E3Requested, addr.into()); - Ok(()) - } - }) - .recipient(), - ); - - info!("Sortition actor started"); - Ok(addr) - } - - pub fn get_nodes(&self, chain_id: u64) -> Result> { - let map = self - .backends - .get() - .ok_or_else(|| anyhow!("Could not get backends cache"))?; - let backend = map - .get(&chain_id) - .ok_or_else(|| anyhow!("No backend for chain_id {}", chain_id))?; - Ok(backend.nodes()) - } - - pub fn get_node_index( - &self, - e3_id: E3id, - seed: Seed, - size: usize, - chain_id: u64, - ) -> Option<(u64, Option)> { - let bus = self.bus.clone(); - let map = self.backends.get()?; - let state_map = self.node_state.get()?; - let backend = map.get(&chain_id)?; - let state = state_map.get(&chain_id)?; - - backend - .get_index(e3_id, seed, size, self.address.clone(), chain_id, state) - .unwrap_or_else(|err| { - bus.err(EType::Sortition, err); - None - }) - } - - fn get_committee(&self, e3_id: &E3id) -> Option { - self.finalized_committees - .get() - .and_then(|committees| committees.get(e3_id).cloned()) - } - - /// Resolve an expelled node's `party_id` against the finalized committee and re-publish the - /// enriched [`CommitteeMemberExpelled`] event for downstream actors. - /// - /// Returns `Ok(true)` when the committee is known (the expulsion was handled, whether or not - /// the node was a member) and `Ok(false)` when the committee has not been finalized yet, in - /// which case the caller should buffer the event and retry after finalization (C18). - fn try_resolve_and_publish_expulsion( - &self, - data: CommitteeMemberExpelled, - ec: EventContext, - ) -> Result { - let node_addr = data.node.to_string(); - - let Some(committee) = self.get_committee(&data.e3_id) else { - return Ok(false); - }; - - let Some(party_id) = committee.party_id_for(&node_addr) else { - warn!( - "Expelled node {} not found in committee for e3_id={}", - node_addr, data.e3_id - ); - return Ok(true); - }; - - info!( - "Sortition: resolved expelled node {} to party_id={} for e3_id={}, re-publishing enriched event", - node_addr, party_id, data.e3_id - ); - - self.bus.publish( - CommitteeMemberExpelled { - party_id: Some(party_id), - ..data - }, - ec, - )?; - - Ok(true) - } - - fn committee_contains(&mut self, e3_id: E3id, node: String) -> bool { - let Some(committee) = self.get_committee(&e3_id) else { - // Non blocking error - self.bus.err( - EType::Sortition, - anyhow!("No finalized committee found for E3 {}", e3_id), - ); - return false; - }; - - committee.contains(&node) - } - /// Helper method to release active jobs for an E3's committee. - fn decrement_jobs_for_e3( - &mut self, - e3_id: &E3id, - reason: &str, - ec: EventContext, - ) -> Result<()> { - self.node_state.try_mutate(&ec, |mut state_map| { - NodeRegistry::release_committee_jobs(&mut state_map, e3_id, reason); - Ok(state_map) - }) - } -} - -impl Actor for Sortition { - type Context = actix::Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT); - } -} - -impl Handler for Sortition { - type Result = (); - - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - match msg { - InterfoldEventData::E3Requested(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::CiphernodeAdded(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::CiphernodeRemoved(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::TicketBalanceUpdated(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::OperatorActivationChanged(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ConfigurationUpdated(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::CommitteePublished(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::PlaintextOutputPublished(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::CommitteeFinalized(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::CommitteeMemberExpelled(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::E3Failed(data) => self.notify_sync(ctx, TypedEvent::new(data, ec)), - InterfoldEventData::E3StageChanged(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::E3RequestComplete(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - _ => (), - } - } -} - -impl Handler> for Sortition { - type Result = (); - fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { - let e3_id = msg.e3_id.clone(); - let chain_id = msg.e3_id.chain_id(); - let seed = msg.seed; - let threshold_m = msg.threshold_m; - let threshold_n = msg.threshold_n; - let buffer = ticket_sortition::calculate_buffer_size(threshold_m, threshold_n); - let total_selection_size = threshold_n + buffer; - - info!( - e3_id = %e3_id, - threshold_m = threshold_m, - threshold_n = threshold_n, - buffer = buffer, - total_selection_size = total_selection_size, - "Performing Sortition with buffer" - ); - - let node_index = self.get_node_index(e3_id.clone(), seed, total_selection_size, chain_id); - - match &node_index { - Some((index, ticket_id)) => { - info!( - e3_id = %e3_id, - node = %self.address, - index = index, - ticket_id = ?ticket_id, - "This node was SELECTED for sortition" - ); - } - None => { - info!( - e3_id = %e3_id, - node = %self.address, - "This node was NOT selected for sortition" - ); - } - } - - self.ciphernode_selector - .do_send(WithSortitionTicket::new(msg, node_index, &self.address)) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - trap(EType::Sortition, &self.bus.with_ec(&ec), || { - let chain_id = msg.chain_id; - let addr = msg.address.clone(); - - self.node_state.try_mutate(&ec, |mut state_map| { - NodeRegistry::add_node(&mut state_map, chain_id, addr.clone()); - Ok(state_map) - })?; - self.backends.try_mutate(&ec, move |mut list_map| { - let default_backend = list_map - .get(&u64::MAX) - .cloned() - .unwrap_or_else(SortitionBackend::score); - - list_map - .entry(chain_id) - .or_insert_with(|| default_backend) - .add(addr); - Ok(list_map) - })?; - Ok(()) - }) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - let (msg, ec) = msg.into_components(); - trap(EType::Sortition, &self.bus.with_ec(&ec), || { - let chain_id = msg.chain_id; - let addr = msg.address.clone(); - - self.node_state.try_mutate(&ec, |mut state_map| { - NodeRegistry::remove_node(&mut state_map, chain_id, &addr); - Ok(state_map) - })?; - self.backends.try_mutate(&ec, move |mut list_map| { - if let Some(backend) = list_map.get_mut(&chain_id) { - backend.remove(addr); - } - Ok(list_map) - })?; - Ok(()) - }) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - let (msg, ec) = msg.into_components(); - trap(EType::Sortition, &self.bus.with_ec(&ec), || { - self.node_state.try_mutate(&ec, |mut state_map| { - NodeRegistry::set_ticket_balance( - &mut state_map, - msg.chain_id, - msg.operator.clone(), - msg.new_balance, - ); - Ok(state_map) - }) - }) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - let (msg, ec) = msg.into_components(); - trap(EType::Sortition, &self.bus.with_ec(&ec), || { - self.node_state.try_mutate(&ec, |mut state_map| { - NodeRegistry::set_operator_active(&mut state_map, msg.operator.clone(), msg.active); - Ok(state_map) - }) - }) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - let (msg, ec) = msg.into_components(); - trap(EType::Sortition, &self.bus.with_ec(&ec), || { - if msg.parameter == "ticketPrice" { - self.node_state.try_mutate(&ec, |mut state_map| { - NodeRegistry::set_ticket_price(&mut state_map, msg.chain_id, msg.new_value); - Ok(state_map) - })?; - } - Ok(()) - }) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _: &mut Self::Context, - ) -> Self::Result { - let (msg, ec) = msg.into_components(); - trap(EType::Sortition, &self.bus.with_ec(&ec), || { - self.node_state.try_mutate(&ec, |mut state_map| { - NodeRegistry::record_committee_published(&mut state_map, &msg.e3_id, &msg.nodes); - Ok(state_map) - }) - }) - } -} - -impl Handler for Sortition { - type Result = (); - - fn handle(&mut self, msg: GetCommitteeMembersRequest, _: &mut Self::Context) -> Self::Result { - trap(EType::Sortition, &self.bus.clone(), || { - let members = self.get_committee(&msg.e3_id).map(|c| c.members().to_vec()); - let reply = msg.reply; - // `try_send` can drop the reply when the aggregator mailbox is busy (e.g. mid - // `AggregationProofSigned`), leaving decryption stuck after C7 with no ZK job. - actix::spawn(async move { - if reply - .send(CommitteeMembersResponse { members }) - .await - .is_err() - { - tracing::error!("committee members reply failed: aggregator recipient closed"); - } - }); - Ok(()) - }) - } -} - -impl Handler> for Sortition -where - T: Clone + Send + Sync + 'static, -{ - type Result = (); - fn handle( - &mut self, - msg: E3CommitteeContainsRequest, - _: &mut Self::Context, - ) -> Self::Result { - trap(EType::Sortition, &self.bus.clone(), || { - let response = E3CommitteeContainsResponse::new( - msg.inner, - self.committee_contains(msg.e3_id, msg.node), - ); - msg.sender.try_send(response)?; - Ok(()) - }) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - let (msg, ec) = msg.into_components(); - trap(EType::Sortition, &self.bus.with_ec(&ec), || { - self.decrement_jobs_for_e3(&msg.e3_id, "PlaintextOutputPublished", ec) - }) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle(&mut self, msg: TypedEvent, _ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - trap(EType::Sortition, &self.bus.with_ec(&ec), || { - let reason = format!("E3Failed: {:?}", msg.reason); - self.decrement_jobs_for_e3(&msg.e3_id, &reason, ec) - }) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - let (msg, ec) = msg.into_components(); - trap(EType::Sortition, &self.bus.with_ec(&ec), || { - match msg.new_stage { - E3Stage::Complete | E3Stage::Failed => { - let reason = format!("E3StageChanged to {:?}", msg.new_stage); - self.decrement_jobs_for_e3(&msg.e3_id, &reason, ec)?; - } - _ => { - // Non-terminal stages, no action needed - } - } - Ok(()) - }) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - trap(EType::Sortition, &self.bus.with_ec(msg.get_ctx()), || { - self.finalized_committees - .try_mutate(msg.get_ctx(), |mut committees| { - FinalizedCommitteeRetention::remove(&mut committees, &msg.e3_id); - Ok(committees) - })?; - self.pending_expulsions.remove(&msg.e3_id); - Ok(()) - }) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - let (mut msg, ec) = msg.into_components(); - msg.sort_by_score(); - trap(EType::Sortition, &self.bus.with_ec(&ec), || { - info!( - e3_id = %msg.e3_id, - committee_size = msg.committee.len(), - "Storing finalized committee" - ); - - self.finalized_committees - .try_mutate(&ec, |mut committees| { - committees.insert(msg.e3_id.clone(), Committee::new(msg.committee.clone())); - Ok(committees) - })?; - - // Drain any expulsions that arrived before the committee was finalized (C18). - if let Some(buffered) = self.pending_expulsions.remove(&msg.e3_id) { - info!( - e3_id = %msg.e3_id, - count = buffered.len(), - "Sortition: draining buffered pre-finalization expulsion(s)" - ); - for (data, buffered_ec) in buffered { - if let Err(e) = self.try_resolve_and_publish_expulsion(data, buffered_ec) { - warn!( - e3_id = %msg.e3_id, - error = %e, - "Sortition: failed to process buffered expulsion after finalization" - ); - } - } - } - - Ok(()) - }) - } -} - -impl Handler> for Sortition { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - let (data, ec) = msg.into_components(); - - // Only process raw events from chain (party_id not yet resolved). - // Events we re-publish with party_id set will also arrive here; ignore them. - if data.party_id.is_some() { - return; - } - - trap(EType::Sortition, &self.bus.with_ec(&ec), || { - if self.try_resolve_and_publish_expulsion(data.clone(), ec.clone())? { - return Ok(()); - } - - // Committee not finalized yet — buffer until CommitteeFinalized arrives (C18) instead - // of dropping the expulsion, which would otherwise leave a known-bad member in the - // committee until the round times out. - warn!( - node = %data.node, - e3_id = %data.e3_id, - "CommitteeMemberExpelled arrived before committee finalized; buffering until finalization" - ); - self.pending_expulsions - .entry(data.e3_id.clone()) - .or_default() - .push((data, ec)); - Ok(()) - }) - } -} diff --git a/crates/sortition/src/ciphernode_selection/actor.rs b/crates/sortition/src/ciphernode_selection/actor.rs new file mode 100644 index 0000000000..818625525d --- /dev/null +++ b/crates/sortition/src/ciphernode_selection/actor.rs @@ -0,0 +1,148 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::WithSortitionTicket; +use actix::prelude::*; +use anyhow::bail; +use anyhow::Result; +use e3_data::{AutoPersist, Persistable, Repository}; +use e3_events::E3RequestComplete; +use e3_events::EventContext; +use e3_events::Sequenced; +use e3_events::TypedEvent; +use e3_events::{ + prelude::*, trap, AggregatorChanged, BusHandle, CiphernodeSelected, Committee, + CommitteeFinalized, CommitteeMemberExpelled, E3Requested, E3id, EType, EventType, + InterfoldEvent, InterfoldEventData, Shutdown, TicketGenerated, TicketId, +}; +use e3_request::E3Meta; +use e3_utils::NotifySync; +use e3_utils::MAILBOX_LIMIT; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use tracing::info; + +#[path = "handlers.rs"] +mod handlers; + +/// Build an `E3Meta` from an `E3Requested` event's fields. +fn e3_meta_from(req: &E3Requested) -> E3Meta { + E3Meta { + seed: req.seed, + threshold_n: req.threshold_n, + threshold_m: req.threshold_m, + params_preset: req.params_preset, + params: req.params.clone(), + error_size: req.error_size.clone(), + proof_aggregation_enabled: req.proof_aggregation_enabled, + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, Default)] +pub struct CiphernodeSelectorState { + pub e3_cache: HashMap, + pub committees: HashMap, + pub expelled: HashMap>, + /// Party ids the local node presumes unresponsive for failover purposes. + /// Treated identically to `expelled` when computing the active aggregator, + /// but populated by local liveness signals rather than on-chain expulsion. + /// Empty by default, so behaviour is unchanged unless a standby is promoted. + pub unresponsive: HashMap>, + pub is_aggregator: HashMap, +} + +#[derive(Message, Debug, Clone, Copy)] +#[rtype(result = "()")] +pub struct EmitPersistedAggregatorState; + +/// CiphernodeSelector is an actor that determines if a ciphernode is part of a committee and if so +/// emits a TicketGenerated event (score sortition) to the event bus +pub struct CiphernodeSelector { + bus: BusHandle, + address: String, + state: Persistable, +} + +impl Actor for CiphernodeSelector { + type Context = Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } +} + +impl CiphernodeSelector { + pub fn new( + bus: &BusHandle, + state: Persistable, + address: &str, + ) -> Self { + Self { + bus: bus.clone(), + state, + address: address.to_owned(), + } + } + + pub async fn attach( + bus: &BusHandle, + selector_store: Repository, + address: &str, + ) -> Result> { + let state = selector_store + .load_or_default(CiphernodeSelectorState::default()) + .await?; + let addr = CiphernodeSelector::new(bus, state, address).start(); + + bus.subscribe(EventType::E3Requested, addr.clone().recipient()); + bus.subscribe(EventType::E3RequestComplete, addr.clone().recipient()); + bus.subscribe(EventType::CommitteeFinalized, addr.clone().recipient()); + bus.subscribe(EventType::CommitteeMemberExpelled, addr.clone().recipient()); + bus.subscribe(EventType::Shutdown, addr.clone().recipient()); + + info!("CiphernodeSelector listening!"); + Ok(addr) + } + + fn update_aggregator_status( + &mut self, + e3_id: &E3id, + ec: &EventContext, + force_emit: bool, + ) -> Result<()> { + let Some(state) = self.state.get() else { + bail!("Could not get selector state"); + }; + + let committee = state + .committees + .get(e3_id) + .cloned() + .ok_or_else(|| anyhow::anyhow!("Missing finalized committee for {}", e3_id))?; + let expelled = state.expelled.get(e3_id).cloned().unwrap_or_default(); + let unresponsive = state.unresponsive.get(e3_id).cloned().unwrap_or_default(); + let is_aggregator = committee.effective_aggregator(&self.address, &expelled, &unresponsive); + let previous = state.is_aggregator.get(e3_id).copied(); + + self.state.try_mutate(ec, |mut selector_state| { + selector_state + .is_aggregator + .insert(e3_id.clone(), is_aggregator); + Ok(selector_state) + })?; + + if force_emit || previous != Some(is_aggregator) { + self.bus.publish( + AggregatorChanged { + e3_id: e3_id.clone(), + is_aggregator, + }, + ec.clone(), + )?; + } + + Ok(()) + } +} diff --git a/crates/sortition/src/actors/ciphernode_selector.rs b/crates/sortition/src/ciphernode_selection/handlers.rs similarity index 64% rename from crates/sortition/src/actors/ciphernode_selector.rs rename to crates/sortition/src/ciphernode_selection/handlers.rs index fe990a536b..e2f7479e42 100644 --- a/crates/sortition/src/actors/ciphernode_selector.rs +++ b/crates/sortition/src/ciphernode_selection/handlers.rs @@ -1,148 +1,8 @@ // SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. -use crate::WithSortitionTicket; -use actix::prelude::*; -use anyhow::bail; -use anyhow::Result; -use e3_data::{AutoPersist, Persistable, Repository}; -use e3_events::E3RequestComplete; -use e3_events::EventContext; -use e3_events::Sequenced; -use e3_events::TypedEvent; -use e3_events::{ - prelude::*, trap, AggregatorChanged, BusHandle, CiphernodeSelected, Committee, - CommitteeFinalized, CommitteeMemberExpelled, E3Requested, E3id, EType, EventType, - InterfoldEvent, InterfoldEventData, Shutdown, TicketGenerated, TicketId, -}; -use e3_request::E3Meta; -use e3_utils::NotifySync; -use e3_utils::MAILBOX_LIMIT; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use tracing::info; +//! Selector event routing and lifecycle handlers. -/// Build an `E3Meta` from an `E3Requested` event's fields. -fn e3_meta_from(req: &E3Requested) -> E3Meta { - E3Meta { - seed: req.seed, - threshold_n: req.threshold_n, - threshold_m: req.threshold_m, - params_preset: req.params_preset, - params: req.params.clone(), - error_size: req.error_size.clone(), - proof_aggregation_enabled: req.proof_aggregation_enabled, - } -} - -#[derive(Clone, Debug, Serialize, Deserialize, Default)] -pub struct CiphernodeSelectorState { - pub e3_cache: HashMap, - pub committees: HashMap, - pub expelled: HashMap>, - /// Party ids the local node presumes unresponsive for failover purposes. - /// Treated identically to `expelled` when computing the active aggregator, - /// but populated by local liveness signals rather than on-chain expulsion. - /// Empty by default, so behaviour is unchanged unless a standby is promoted. - pub unresponsive: HashMap>, - pub is_aggregator: HashMap, -} - -#[derive(Message, Debug, Clone, Copy)] -#[rtype(result = "()")] -pub struct EmitPersistedAggregatorState; - -/// CiphernodeSelector is an actor that determines if a ciphernode is part of a committee and if so -/// emits a TicketGenerated event (score sortition) to the event bus -pub struct CiphernodeSelector { - bus: BusHandle, - address: String, - state: Persistable, -} - -impl Actor for CiphernodeSelector { - type Context = Context; - fn started(&mut self, ctx: &mut Self::Context) { - ctx.set_mailbox_capacity(MAILBOX_LIMIT); - } -} - -impl CiphernodeSelector { - pub fn new( - bus: &BusHandle, - state: Persistable, - address: &str, - ) -> Self { - Self { - bus: bus.clone(), - state, - address: address.to_owned(), - } - } - - pub async fn attach( - bus: &BusHandle, - selector_store: Repository, - address: &str, - ) -> Result> { - let state = selector_store - .load_or_default(CiphernodeSelectorState::default()) - .await?; - let addr = CiphernodeSelector::new(bus, state, address).start(); - - bus.subscribe(EventType::E3Requested, addr.clone().recipient()); - bus.subscribe(EventType::E3RequestComplete, addr.clone().recipient()); - bus.subscribe(EventType::CommitteeFinalized, addr.clone().recipient()); - bus.subscribe(EventType::CommitteeMemberExpelled, addr.clone().recipient()); - bus.subscribe(EventType::Shutdown, addr.clone().recipient()); - - info!("CiphernodeSelector listening!"); - Ok(addr) - } - - fn update_aggregator_status( - &mut self, - e3_id: &E3id, - ec: &EventContext, - force_emit: bool, - ) -> Result<()> { - let Some(state) = self.state.get() else { - bail!("Could not get selector state"); - }; - - let committee = state - .committees - .get(e3_id) - .cloned() - .ok_or_else(|| anyhow::anyhow!("Missing finalized committee for {}", e3_id))?; - let expelled = state.expelled.get(e3_id).cloned().unwrap_or_default(); - let unresponsive = state.unresponsive.get(e3_id).cloned().unwrap_or_default(); - let is_aggregator = committee.effective_aggregator(&self.address, &expelled, &unresponsive); - let previous = state.is_aggregator.get(e3_id).copied(); - - self.state.try_mutate(ec, |mut selector_state| { - selector_state - .is_aggregator - .insert(e3_id.clone(), is_aggregator); - Ok(selector_state) - })?; - - if force_emit || previous != Some(is_aggregator) { - self.bus.publish( - AggregatorChanged { - e3_id: e3_id.clone(), - is_aggregator, - }, - ec.clone(), - )?; - } - - Ok(()) - } -} +use super::*; impl Handler for CiphernodeSelector { type Result = (); diff --git a/crates/sortition/src/domain/mod.rs b/crates/sortition/src/domain.rs similarity index 61% rename from crates/sortition/src/domain/mod.rs rename to crates/sortition/src/domain.rs index dd9f37bac8..e7f7c16c8e 100644 --- a/crates/sortition/src/domain/mod.rs +++ b/crates/sortition/src/domain.rs @@ -4,10 +4,17 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +//! Compatibility view of pure sortition modules stored by capability. + +#[path = "sortition/selection_backend.rs"] pub mod backends; +#[path = "sortition/finalized_committee_retention.rs"] mod finalized_committee_retention; +#[path = "sortition/node_registry.rs"] pub mod node_registry; +#[path = "sortition/ticket.rs"] pub mod ticket; +#[path = "sortition/ticket_selection.rs"] pub mod ticket_sortition; pub use backends::*; diff --git a/crates/sortition/src/sortition/actor.rs b/crates/sortition/src/sortition/actor.rs new file mode 100644 index 0000000000..e2aed4e6a3 --- /dev/null +++ b/crates/sortition/src/sortition/actor.rs @@ -0,0 +1,254 @@ +// SPDX-License-Identifier: LGPL-4.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use crate::domain::backends::{SortitionBackend, SortitionList}; +use crate::domain::node_registry::{NodeRegistry, NodeStateStore}; +use crate::domain::ticket_sortition; +use crate::messages::{ + CommitteeMembersResponse, E3CommitteeContainsRequest, E3CommitteeContainsResponse, + GetCommitteeMembersRequest, WithSortitionTicket, +}; +use crate::CiphernodeSelector; +use crate::FinalizedCommitteeRetention; +use actix::prelude::*; +use anyhow::{anyhow, Result}; +use e3_data::{AutoPersist, Persistable, Repository}; +use e3_events::{ + prelude::*, trap, CiphernodeAdded, CiphernodeRemoved, Committee, CommitteeFinalized, + CommitteeMemberExpelled, CommitteePublished, ConfigurationUpdated, E3Failed, E3RequestComplete, + E3Requested, E3Stage, E3StageChanged, EType, EventContext, EventType, InterfoldEvent, + OperatorActivationChanged, PlaintextOutputPublished, Seed, Sequenced, TicketBalanceUpdated, + TypedEvent, +}; +use e3_events::{BusHandle, E3id, InterfoldEventData}; +use e3_utils::{NotifySync, MAILBOX_LIMIT}; +use std::collections::HashMap; +use tracing::{info, instrument, warn}; + +/// Sortition actor that manages the sortition algorithm and the node state. +pub struct Sortition { + /// Persistent map of `chain_id -> SortitionBackend`. + backends: Persistable>, + /// Persistent map of `chain_id -> NodeStateStore`. + node_state: Persistable>, + /// Event bus for error reporting and interfold event subscription. + bus: BusHandle, + /// Persistent map of finalized committees per E3 + finalized_committees: Persistable>, + /// Address for the CiphernodeSelector + ciphernode_selector: Addr, + /// Address for the current node + address: String, + /// Ephemeral buffer of raw `CommitteeMemberExpelled` events that arrived before the matching + /// committee was finalized (e.g. out-of-order live delivery or a reorg). Drained when the + /// `CommitteeFinalized` event for the same E3 is processed so early expulsions are not lost. + pending_expulsions: HashMap)>>, +} + +/// Parameters for constructing a `Sortition` actor. +#[derive(Debug)] +pub struct SortitionParams { + /// Event bus address. + pub bus: BusHandle, + /// Persisted per-chain backend map. + pub backends: Persistable>, + /// Node state store per chain + pub node_state: Persistable>, + /// Persistent map of finalized committees per E3 + pub finalized_committees: Persistable>, + /// Address for the CiphernodeSelector + pub ciphernode_selector: Addr, + /// Address for the current node + pub address: String, +} + +impl Sortition { + pub fn new(params: SortitionParams) -> Self { + Self { + backends: params.backends, + node_state: params.node_state, + bus: params.bus, + finalized_committees: params.finalized_committees, + ciphernode_selector: params.ciphernode_selector, + address: params.address, + pending_expulsions: HashMap::new(), + } + } + + #[instrument(name = "sortition_attach", skip_all)] + pub async fn attach( + bus: &BusHandle, + backends_store: Repository>, + node_state_store: Repository>, + committees_store: Repository>, + default_backend: SortitionBackend, + ciphernode_selector: Addr, + address: &str, + ) -> Result> { + let mut backends = backends_store.load_or_default(HashMap::new()).await?; + let node_state = node_state_store.load_or_default(HashMap::new()).await?; + let finalized_committees = committees_store.load_or_default(HashMap::new()).await?; + + backends.try_mutate_without_context(|mut list| { + list.insert(u64::MAX, default_backend); + Ok(list) + })?; + + let addr = Sortition::new(SortitionParams { + bus: bus.clone(), + backends, + node_state, + finalized_committees, + ciphernode_selector, + address: address.to_owned(), + }) + .start(); + + // Subscribe to state-building events immediately (needed during EventStore replay) + bus.subscribe_all( + &[ + EventType::CiphernodeAdded, + EventType::CiphernodeRemoved, + EventType::TicketBalanceUpdated, + EventType::OperatorActivationChanged, + EventType::ConfigurationUpdated, + EventType::CommitteePublished, + EventType::PlaintextOutputPublished, + EventType::CommitteeFinalized, + EventType::CommitteeMemberExpelled, + EventType::E3Failed, + EventType::E3StageChanged, + EventType::E3RequestComplete, + ], + addr.clone().into(), + ); + + // Gate E3Requested behind EffectsEnabled — sortition should not trigger + // ticket generation during historical event replay. + bus.subscribe( + EventType::EffectsEnabled, + e3_events::run_once::({ + let bus = bus.clone(); + let addr = addr.clone(); + move |_| { + bus.subscribe(EventType::E3Requested, addr.into()); + Ok(()) + } + }) + .recipient(), + ); + + info!("Sortition actor started"); + Ok(addr) + } + + pub fn get_nodes(&self, chain_id: u64) -> Result> { + let map = self + .backends + .get() + .ok_or_else(|| anyhow!("Could not get backends cache"))?; + let backend = map + .get(&chain_id) + .ok_or_else(|| anyhow!("No backend for chain_id {}", chain_id))?; + Ok(backend.nodes()) + } + + pub fn get_node_index( + &self, + e3_id: E3id, + seed: Seed, + size: usize, + chain_id: u64, + ) -> Option<(u64, Option)> { + let bus = self.bus.clone(); + let map = self.backends.get()?; + let state_map = self.node_state.get()?; + let backend = map.get(&chain_id)?; + let state = state_map.get(&chain_id)?; + + backend + .get_index(e3_id, seed, size, self.address.clone(), chain_id, state) + .unwrap_or_else(|err| { + bus.err(EType::Sortition, err); + None + }) + } + + fn get_committee(&self, e3_id: &E3id) -> Option { + self.finalized_committees + .get() + .and_then(|committees| committees.get(e3_id).cloned()) + } + + /// Resolve an expelled node's `party_id` against the finalized committee and re-publish the + /// enriched [`CommitteeMemberExpelled`] event for downstream actors. + /// + /// Returns `Ok(true)` when the committee is known (the expulsion was handled, whether or not + /// the node was a member) and `Ok(false)` when the committee has not been finalized yet, in + /// which case the caller should buffer the event and retry after finalization (C18). + fn try_resolve_and_publish_expulsion( + &self, + data: CommitteeMemberExpelled, + ec: EventContext, + ) -> Result { + let node_addr = data.node.to_string(); + + let Some(committee) = self.get_committee(&data.e3_id) else { + return Ok(false); + }; + + let Some(party_id) = committee.party_id_for(&node_addr) else { + warn!( + "Expelled node {} not found in committee for e3_id={}", + node_addr, data.e3_id + ); + return Ok(true); + }; + + info!( + "Sortition: resolved expelled node {} to party_id={} for e3_id={}, re-publishing enriched event", + node_addr, party_id, data.e3_id + ); + + self.bus.publish( + CommitteeMemberExpelled { + party_id: Some(party_id), + ..data + }, + ec, + )?; + + Ok(true) + } + + fn committee_contains(&mut self, e3_id: E3id, node: String) -> bool { + let Some(committee) = self.get_committee(&e3_id) else { + // Non blocking error + self.bus.err( + EType::Sortition, + anyhow!("No finalized committee found for E3 {}", e3_id), + ); + return false; + }; + + committee.contains(&node) + } + /// Helper method to release active jobs for an E3's committee. + fn decrement_jobs_for_e3( + &mut self, + e3_id: &E3id, + reason: &str, + ec: EventContext, + ) -> Result<()> { + self.node_state.try_mutate(&ec, |mut state_map| { + NodeRegistry::release_committee_jobs(&mut state_map, e3_id, reason); + Ok(state_map) + }) + } +} + +#[path = "handlers/mod.rs"] +mod handlers; diff --git a/crates/sortition/src/domain/finalized_committee_retention.rs b/crates/sortition/src/sortition/finalized_committee_retention.rs similarity index 100% rename from crates/sortition/src/domain/finalized_committee_retention.rs rename to crates/sortition/src/sortition/finalized_committee_retention.rs diff --git a/crates/sortition/src/sortition/handlers/committee.rs b/crates/sortition/src/sortition/handlers/committee.rs new file mode 100644 index 0000000000..fb724d2ea8 --- /dev/null +++ b/crates/sortition/src/sortition/handlers/committee.rs @@ -0,0 +1,89 @@ +// SPDX-License-Identifier: LGPL-4.0-only + +//! Persist finalized committees and enrich expulsion facts. + +use super::*; + +impl Handler> for Sortition { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (mut msg, ec) = msg.into_components(); + msg.sort_by_score(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + info!( + e3_id = %msg.e3_id, + committee_size = msg.committee.len(), + "Storing finalized committee" + ); + + self.finalized_committees + .try_mutate(&ec, |mut committees| { + committees.insert(msg.e3_id.clone(), Committee::new(msg.committee.clone())); + Ok(committees) + })?; + + // Drain any expulsions that arrived before the committee was finalized (C18). + if let Some(buffered) = self.pending_expulsions.remove(&msg.e3_id) { + info!( + e3_id = %msg.e3_id, + count = buffered.len(), + "Sortition: draining buffered pre-finalization expulsion(s)" + ); + for (data, buffered_ec) in buffered { + if let Err(e) = self.try_resolve_and_publish_expulsion(data, buffered_ec) { + warn!( + e3_id = %msg.e3_id, + error = %e, + "Sortition: failed to process buffered expulsion after finalization" + ); + } + } + } + + Ok(()) + }) + } +} + +impl Handler> for Sortition { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (data, ec) = msg.into_components(); + + // Only process raw events from chain (party_id not yet resolved). + // Events we re-publish with party_id set will also arrive here; ignore them. + if data.party_id.is_some() { + return; + } + + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + if self.try_resolve_and_publish_expulsion(data.clone(), ec.clone())? { + return Ok(()); + } + + // Committee not finalized yet — buffer until CommitteeFinalized arrives (C18) instead + // of dropping the expulsion, which would otherwise leave a known-bad member in the + // committee until the round times out. + warn!( + node = %data.node, + e3_id = %data.e3_id, + "CommitteeMemberExpelled arrived before committee finalized; buffering until finalization" + ); + self.pending_expulsions + .entry(data.e3_id.clone()) + .or_default() + .push((data, ec)); + Ok(()) + }) + } +} diff --git a/crates/sortition/src/sortition/handlers/envelope.rs b/crates/sortition/src/sortition/handlers/envelope.rs new file mode 100644 index 0000000000..ac3a77b872 --- /dev/null +++ b/crates/sortition/src/sortition/handlers/envelope.rs @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: LGPL-4.0-only + +//! Actix lifecycle and event-envelope routing. + +use super::*; + +impl Actor for Sortition { + type Context = actix::Context; + fn started(&mut self, ctx: &mut Self::Context) { + ctx.set_mailbox_capacity(MAILBOX_LIMIT); + } +} + +impl Handler for Sortition { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + match msg { + InterfoldEventData::E3Requested(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::CiphernodeAdded(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::CiphernodeRemoved(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::TicketBalanceUpdated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::OperatorActivationChanged(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ConfigurationUpdated(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::CommitteePublished(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::PlaintextOutputPublished(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::CommitteeFinalized(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::CommitteeMemberExpelled(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::E3Failed(data) => self.notify_sync(ctx, TypedEvent::new(data, ec)), + InterfoldEventData::E3StageChanged(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::E3RequestComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + _ => (), + } + } +} diff --git a/crates/sortition/src/sortition/handlers/lifecycle.rs b/crates/sortition/src/sortition/handlers/lifecycle.rs new file mode 100644 index 0000000000..70cc2d8426 --- /dev/null +++ b/crates/sortition/src/sortition/handlers/lifecycle.rs @@ -0,0 +1,138 @@ +// SPDX-License-Identifier: LGPL-4.0-only + +//! Committee queries, job release, and terminal cleanup. + +use super::*; + +impl Handler> for Sortition { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + self.node_state.try_mutate(&ec, |mut state_map| { + NodeRegistry::record_committee_published(&mut state_map, &msg.e3_id, &msg.nodes); + Ok(state_map) + }) + }) + } +} + +impl Handler for Sortition { + type Result = (); + + fn handle(&mut self, msg: GetCommitteeMembersRequest, _: &mut Self::Context) -> Self::Result { + trap(EType::Sortition, &self.bus.clone(), || { + let members = self.get_committee(&msg.e3_id).map(|c| c.members().to_vec()); + let reply = msg.reply; + // `try_send` can drop the reply when the aggregator mailbox is busy (e.g. mid + // `AggregationProofSigned`), leaving decryption stuck after C7 with no ZK job. + actix::spawn(async move { + if reply + .send(CommitteeMembersResponse { members }) + .await + .is_err() + { + tracing::error!("committee members reply failed: aggregator recipient closed"); + } + }); + Ok(()) + }) + } +} + +impl Handler> for Sortition +where + T: Clone + Send + Sync + 'static, +{ + type Result = (); + fn handle( + &mut self, + msg: E3CommitteeContainsRequest, + _: &mut Self::Context, + ) -> Self::Result { + trap(EType::Sortition, &self.bus.clone(), || { + let response = E3CommitteeContainsResponse::new( + msg.inner, + self.committee_contains(msg.e3_id, msg.node), + ); + msg.sender.try_send(response)?; + Ok(()) + }) + } +} + +impl Handler> for Sortition { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + self.decrement_jobs_for_e3(&msg.e3_id, "PlaintextOutputPublished", ec) + }) + } +} + +impl Handler> for Sortition { + type Result = (); + + fn handle(&mut self, msg: TypedEvent, _ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + let reason = format!("E3Failed: {:?}", msg.reason); + self.decrement_jobs_for_e3(&msg.e3_id, &reason, ec) + }) + } +} + +impl Handler> for Sortition { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + match msg.new_stage { + E3Stage::Complete | E3Stage::Failed => { + let reason = format!("E3StageChanged to {:?}", msg.new_stage); + self.decrement_jobs_for_e3(&msg.e3_id, &reason, ec)?; + } + _ => { + // Non-terminal stages, no action needed + } + } + Ok(()) + }) + } +} + +impl Handler> for Sortition { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + trap(EType::Sortition, &self.bus.with_ec(msg.get_ctx()), || { + self.finalized_committees + .try_mutate(msg.get_ctx(), |mut committees| { + FinalizedCommitteeRetention::remove(&mut committees, &msg.e3_id); + Ok(committees) + })?; + self.pending_expulsions.remove(&msg.e3_id); + Ok(()) + }) + } +} diff --git a/crates/sortition/src/sortition/handlers/mod.rs b/crates/sortition/src/sortition/handlers/mod.rs new file mode 100644 index 0000000000..215dd9bc68 --- /dev/null +++ b/crates/sortition/src/sortition/handlers/mod.rs @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: LGPL-4.0-only + +//! Mailbox entry points grouped by message family. + +use super::*; + +mod committee; +mod envelope; +mod lifecycle; +mod registry; +mod request; diff --git a/crates/sortition/src/sortition/handlers/registry.rs b/crates/sortition/src/sortition/handlers/registry.rs new file mode 100644 index 0000000000..2d8c05ec87 --- /dev/null +++ b/crates/sortition/src/sortition/handlers/registry.rs @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: LGPL-4.0-only + +//! Apply node-registry, collateral, activation, and configuration facts. + +use super::*; + +impl Handler> for Sortition { + type Result = (); + + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + let chain_id = msg.chain_id; + let addr = msg.address.clone(); + + self.node_state.try_mutate(&ec, |mut state_map| { + NodeRegistry::add_node(&mut state_map, chain_id, addr.clone()); + Ok(state_map) + })?; + self.backends.try_mutate(&ec, move |mut list_map| { + let default_backend = list_map + .get(&u64::MAX) + .cloned() + .unwrap_or_else(SortitionBackend::score); + + list_map + .entry(chain_id) + .or_insert_with(|| default_backend) + .add(addr); + Ok(list_map) + })?; + Ok(()) + }) + } +} + +impl Handler> for Sortition { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + let chain_id = msg.chain_id; + let addr = msg.address.clone(); + + self.node_state.try_mutate(&ec, |mut state_map| { + NodeRegistry::remove_node(&mut state_map, chain_id, &addr); + Ok(state_map) + })?; + self.backends.try_mutate(&ec, move |mut list_map| { + if let Some(backend) = list_map.get_mut(&chain_id) { + backend.remove(addr); + } + Ok(list_map) + })?; + Ok(()) + }) + } +} + +impl Handler> for Sortition { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + self.node_state.try_mutate(&ec, |mut state_map| { + NodeRegistry::set_ticket_balance( + &mut state_map, + msg.chain_id, + msg.operator.clone(), + msg.new_balance, + ); + Ok(state_map) + }) + }) + } +} + +impl Handler> for Sortition { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + self.node_state.try_mutate(&ec, |mut state_map| { + NodeRegistry::set_operator_active(&mut state_map, msg.operator.clone(), msg.active); + Ok(state_map) + }) + }) + } +} + +impl Handler> for Sortition { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + trap(EType::Sortition, &self.bus.with_ec(&ec), || { + if msg.parameter == "ticketPrice" { + self.node_state.try_mutate(&ec, |mut state_map| { + NodeRegistry::set_ticket_price(&mut state_map, msg.chain_id, msg.new_value); + Ok(state_map) + })?; + } + Ok(()) + }) + } +} diff --git a/crates/sortition/src/sortition/handlers/request.rs b/crates/sortition/src/sortition/handlers/request.rs new file mode 100644 index 0000000000..ba9555c022 --- /dev/null +++ b/crates/sortition/src/sortition/handlers/request.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: LGPL-4.0-only + +//! E3 ticket sortition and selector dispatch. + +use super::*; + +impl Handler> for Sortition { + type Result = (); + fn handle(&mut self, msg: TypedEvent, _: &mut Self::Context) -> Self::Result { + let e3_id = msg.e3_id.clone(); + let chain_id = msg.e3_id.chain_id(); + let seed = msg.seed; + let threshold_m = msg.threshold_m; + let threshold_n = msg.threshold_n; + let buffer = ticket_sortition::calculate_buffer_size(threshold_m, threshold_n); + let total_selection_size = threshold_n + buffer; + + info!( + e3_id = %e3_id, + threshold_m = threshold_m, + threshold_n = threshold_n, + buffer = buffer, + total_selection_size = total_selection_size, + "Performing Sortition with buffer" + ); + + let node_index = self.get_node_index(e3_id.clone(), seed, total_selection_size, chain_id); + + match &node_index { + Some((index, ticket_id)) => { + info!( + e3_id = %e3_id, + node = %self.address, + index = index, + ticket_id = ?ticket_id, + "This node was SELECTED for sortition" + ); + } + None => { + info!( + e3_id = %e3_id, + node = %self.address, + "This node was NOT selected for sortition" + ); + } + } + + self.ciphernode_selector + .do_send(WithSortitionTicket::new(msg, node_index, &self.address)) + } +} diff --git a/crates/sortition/src/domain/node_registry.rs b/crates/sortition/src/sortition/node_registry.rs similarity index 69% rename from crates/sortition/src/domain/node_registry.rs rename to crates/sortition/src/sortition/node_registry.rs index 4793ed555f..1d4f5cfa0c 100644 --- a/crates/sortition/src/domain/node_registry.rs +++ b/crates/sortition/src/sortition/node_registry.rs @@ -281,120 +281,5 @@ pub struct OpenCommittee { } #[cfg(test)] -mod tests { - use super::*; - - fn e3(chain_id: u64, id: &str) -> E3id { - E3id::new(id, chain_id) - } - - #[test] - fn add_and_remove_node() { - let mut store = HashMap::new(); - NodeRegistry::add_node(&mut store, 1, "0xabc".into()); - assert!(store[&1].nodes.contains_key("0xabc")); - - NodeRegistry::remove_node(&mut store, 1, "0xabc"); - assert!(!store[&1].nodes.contains_key("0xabc")); - - // Removing an unknown node / chain is a no-op. - NodeRegistry::remove_node(&mut store, 99, "0xdef"); - } - - #[test] - fn available_tickets_accounts_for_price_and_jobs() { - let mut store = HashMap::new(); - NodeRegistry::set_ticket_price(&mut store, 1, U256::from(10)); - NodeRegistry::set_ticket_balance(&mut store, 1, "0xabc".into(), U256::from(55)); - NodeRegistry::set_operator_active(&mut store, "0xabc".into(), true); - - // floor(55 / 10) = 5 tickets, no active jobs yet. - assert_eq!(store[&1].available_tickets("0xabc"), 5); - - NodeRegistry::record_committee_published(&mut store, &e3(1, "7"), &["0xabc".into()]); - // One active job now -> 4 available. - assert_eq!(store[&1].available_tickets("0xabc"), 4); - assert_eq!(store[&1].nodes["0xabc"].active_jobs, 1); - } - - #[test] - fn zero_price_yields_no_tickets() { - let mut store = HashMap::new(); - NodeRegistry::set_ticket_balance(&mut store, 1, "0xabc".into(), U256::from(100)); - assert_eq!(store[&1].available_tickets("0xabc"), 0); - } - - #[test] - fn operator_active_applies_across_chains() { - let mut store = HashMap::new(); - NodeRegistry::add_node(&mut store, 1, "0xabc".into()); - NodeRegistry::add_node(&mut store, 2, "0xabc".into()); - NodeRegistry::set_operator_active(&mut store, "0xabc".into(), true); - assert!(store[&1].nodes["0xabc"].active); - assert!(store[&2].nodes["0xabc"].active); - } - - #[test] - fn release_committee_jobs_is_idempotent() { - let mut store = HashMap::new(); - let id = e3(1, "7"); - NodeRegistry::record_committee_published( - &mut store, - &id, - &["0xabc".into(), "0xdef".into()], - ); - assert_eq!(store[&1].nodes["0xabc"].active_jobs, 1); - assert_eq!(store[&1].nodes["0xdef"].active_jobs, 1); - - NodeRegistry::release_committee_jobs(&mut store, &id, "test"); - assert_eq!(store[&1].nodes["0xabc"].active_jobs, 0); - assert_eq!(store[&1].nodes["0xdef"].active_jobs, 0); - assert!(!store[&1].e3_committees.contains_key(&committee_key(&id))); - - // Second release does not underflow. - NodeRegistry::release_committee_jobs(&mut store, &id, "test-again"); - assert_eq!(store[&1].nodes["0xabc"].active_jobs, 0); - } - - #[test] - fn get_nodes_with_tickets_filters_inactive_and_empty() { - let mut store = HashMap::new(); - NodeRegistry::set_ticket_price(&mut store, 1, U256::from(10)); - NodeRegistry::set_ticket_balance(&mut store, 1, "active".into(), U256::from(30)); - NodeRegistry::set_operator_active(&mut store, "active".into(), true); - // Inactive node with balance is excluded. - NodeRegistry::set_ticket_balance(&mut store, 1, "inactive".into(), U256::from(30)); - - let with_tickets = store[&1].get_nodes_with_tickets(); - assert_eq!(with_tickets.len(), 1); - assert_eq!(with_tickets[0].0, "active"); - assert_eq!(with_tickets[0].1, 3); - } - - #[test] - fn open_committees_lists_only_unreleased() { - let mut store = HashMap::new(); - let a = e3(1, "1"); - let b = e3(1, "2"); - NodeRegistry::record_committee_published(&mut store, &a, &["0xabc".into()]); - NodeRegistry::record_committee_published(&mut store, &b, &["0xabc".into(), "0xdef".into()]); - - let open = NodeRegistry::open_committees(&store); - assert_eq!(open.len(), 2); - - // Releasing one removes it from the open set. - NodeRegistry::release_committee_jobs(&mut store, &a, "test"); - let open = NodeRegistry::open_committees(&store); - assert_eq!(open.len(), 1); - assert_eq!(open[0].committee_key, committee_key(&b)); - assert_eq!(open[0].chain_id, 1); - assert_eq!( - open[0].members, - vec!["0xabc".to_string(), "0xdef".to_string()] - ); - - // Fully drained -> empty. - NodeRegistry::release_committee_jobs(&mut store, &b, "test"); - assert!(NodeRegistry::open_committees(&store).is_empty()); - } -} +#[path = "node_registry_tests.rs"] +mod tests; diff --git a/crates/sortition/src/sortition/node_registry_tests.rs b/crates/sortition/src/sortition/node_registry_tests.rs new file mode 100644 index 0000000000..fa17617d26 --- /dev/null +++ b/crates/sortition/src/sortition/node_registry_tests.rs @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +fn e3(chain_id: u64, id: &str) -> E3id { + E3id::new(id, chain_id) +} + +#[test] +fn add_and_remove_node() { + let mut store = HashMap::new(); + NodeRegistry::add_node(&mut store, 1, "0xabc".into()); + assert!(store[&1].nodes.contains_key("0xabc")); + + NodeRegistry::remove_node(&mut store, 1, "0xabc"); + assert!(!store[&1].nodes.contains_key("0xabc")); + + // Removing an unknown node / chain is a no-op. + NodeRegistry::remove_node(&mut store, 99, "0xdef"); +} + +#[test] +fn available_tickets_accounts_for_price_and_jobs() { + let mut store = HashMap::new(); + NodeRegistry::set_ticket_price(&mut store, 1, U256::from(10)); + NodeRegistry::set_ticket_balance(&mut store, 1, "0xabc".into(), U256::from(55)); + NodeRegistry::set_operator_active(&mut store, "0xabc".into(), true); + + // floor(55 / 10) = 5 tickets, no active jobs yet. + assert_eq!(store[&1].available_tickets("0xabc"), 5); + + NodeRegistry::record_committee_published(&mut store, &e3(1, "7"), &["0xabc".into()]); + // One active job now -> 4 available. + assert_eq!(store[&1].available_tickets("0xabc"), 4); + assert_eq!(store[&1].nodes["0xabc"].active_jobs, 1); +} + +#[test] +fn zero_price_yields_no_tickets() { + let mut store = HashMap::new(); + NodeRegistry::set_ticket_balance(&mut store, 1, "0xabc".into(), U256::from(100)); + assert_eq!(store[&1].available_tickets("0xabc"), 0); +} + +#[test] +fn operator_active_applies_across_chains() { + let mut store = HashMap::new(); + NodeRegistry::add_node(&mut store, 1, "0xabc".into()); + NodeRegistry::add_node(&mut store, 2, "0xabc".into()); + NodeRegistry::set_operator_active(&mut store, "0xabc".into(), true); + assert!(store[&1].nodes["0xabc"].active); + assert!(store[&2].nodes["0xabc"].active); +} + +#[test] +fn release_committee_jobs_is_idempotent() { + let mut store = HashMap::new(); + let id = e3(1, "7"); + NodeRegistry::record_committee_published(&mut store, &id, &["0xabc".into(), "0xdef".into()]); + assert_eq!(store[&1].nodes["0xabc"].active_jobs, 1); + assert_eq!(store[&1].nodes["0xdef"].active_jobs, 1); + + NodeRegistry::release_committee_jobs(&mut store, &id, "test"); + assert_eq!(store[&1].nodes["0xabc"].active_jobs, 0); + assert_eq!(store[&1].nodes["0xdef"].active_jobs, 0); + assert!(!store[&1].e3_committees.contains_key(&committee_key(&id))); + + // Second release does not underflow. + NodeRegistry::release_committee_jobs(&mut store, &id, "test-again"); + assert_eq!(store[&1].nodes["0xabc"].active_jobs, 0); +} + +#[test] +fn get_nodes_with_tickets_filters_inactive_and_empty() { + let mut store = HashMap::new(); + NodeRegistry::set_ticket_price(&mut store, 1, U256::from(10)); + NodeRegistry::set_ticket_balance(&mut store, 1, "active".into(), U256::from(30)); + NodeRegistry::set_operator_active(&mut store, "active".into(), true); + // Inactive node with balance is excluded. + NodeRegistry::set_ticket_balance(&mut store, 1, "inactive".into(), U256::from(30)); + + let with_tickets = store[&1].get_nodes_with_tickets(); + assert_eq!(with_tickets.len(), 1); + assert_eq!(with_tickets[0].0, "active"); + assert_eq!(with_tickets[0].1, 3); +} + +#[test] +fn open_committees_lists_only_unreleased() { + let mut store = HashMap::new(); + let a = e3(1, "1"); + let b = e3(1, "2"); + NodeRegistry::record_committee_published(&mut store, &a, &["0xabc".into()]); + NodeRegistry::record_committee_published(&mut store, &b, &["0xabc".into(), "0xdef".into()]); + + let open = NodeRegistry::open_committees(&store); + assert_eq!(open.len(), 2); + + // Releasing one removes it from the open set. + NodeRegistry::release_committee_jobs(&mut store, &a, "test"); + let open = NodeRegistry::open_committees(&store); + assert_eq!(open.len(), 1); + assert_eq!(open[0].committee_key, committee_key(&b)); + assert_eq!(open[0].chain_id, 1); + assert_eq!( + open[0].members, + vec!["0xabc".to_string(), "0xdef".to_string()] + ); + + // Fully drained -> empty. + NodeRegistry::release_committee_jobs(&mut store, &b, "test"); + assert!(NodeRegistry::open_committees(&store).is_empty()); +} diff --git a/crates/sortition/src/domain/backends.rs b/crates/sortition/src/sortition/selection_backend.rs similarity index 100% rename from crates/sortition/src/domain/backends.rs rename to crates/sortition/src/sortition/selection_backend.rs diff --git a/crates/sortition/src/domain/ticket.rs b/crates/sortition/src/sortition/ticket.rs similarity index 100% rename from crates/sortition/src/domain/ticket.rs rename to crates/sortition/src/sortition/ticket.rs diff --git a/crates/sortition/src/domain/ticket_sortition.rs b/crates/sortition/src/sortition/ticket_selection.rs similarity index 100% rename from crates/sortition/src/domain/ticket_sortition.rs rename to crates/sortition/src/sortition/ticket_selection.rs diff --git a/crates/sync/src/actors/sync.rs b/crates/sync/src/actors/sync.rs deleted file mode 100644 index fe9d8997b0..0000000000 --- a/crates/sync/src/actors/sync.rs +++ /dev/null @@ -1,922 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -#[cfg(test)] -use crate::domain::ReplayDecision; -use crate::domain::{ - decide_schema_version, CollectOutcome, HistoricalEvmCollector, SchemaVersionDecision, - SnapshotMeta, SyncPlanner, SCHEMA_VERSION, -}; -use crate::replay_spool::ReplaySpool; -use crate::SyncRepositoryFactory; -use actix::{Message, Recipient}; -use anyhow::{bail, Context, Result}; -use e3_data::Repositories; -use e3_events::{ - AggregateConfig, BusHandle, CorrelationId, EffectsEnabled, Event, EventPublisher, - EventStoreQueryBy, EventStoreQueryResponse, EventSubscriber, EventType, EvmEventConfig, - HistoricalEvmEventsReceived, HistoricalEvmSyncStart, HistoricalNetSyncStart, InterfoldEvent, - InterfoldEventData, SeqAgg, StoreKeys, SyncEnded, Unsequenced, -}; -#[cfg(test)] -use e3_events::{EventBusBarrier, EventBusFanout, EventContextAccessors}; -use e3_utils::actix::channel as actix_toolbox; -use std::time::Duration; -use tokio::sync::mpsc::Receiver; -use tracing::info; - -#[cfg(test)] -const REPLAY_PROGRESS_INTERVAL: usize = 10_000; - -pub async fn sync( - bus: &BusHandle, - default_config: &EvmEventConfig, - repositories: &Repositories, - aggregate_config: &AggregateConfig, - eventstore: &Recipient>, -) -> Result<()> { - // 0. start listening early for net ready - let net_ready = bus.wait_for(EventType::NetReady); - - // 0b. Verify the on-disk schema version is compatible with this binary - // before touching any persisted state, so an incompatible upgrade or - // downgrade halts loudly instead of silently loading garbage (H19/H20). - preflight_schema_version(repositories, aggregate_config, eventstore).await?; - - // 1. Load snapsshot metadata - info!("Loading snapshot metadata..."); - let snapshot = - SnapshotMeta::read_from_disk(aggregate_config.aggregates(), default_config, repositories) - .await?; - info!( - "Snapshot metadata loaded for {} aggregates.", - snapshot.aggregates().len() - ); - - // 1b. Restore the HLC ordering floor from the highest persisted aggregate - // timestamp so events created after this restart remain strictly after - // durable history, including its logical counter, even if wall time moved backwards. - if let Some(max_ts) = snapshot.to_net_config().values().copied().max() { - bus.seed_clock(max_ts)?; - } - - // 2. Determine the evm blocks to read from based on the SnapshotMeta - let evm_config = snapshot.to_evm_config(); - let snapshot_net_config = snapshot.to_net_config(); - - // 3. Page post-snapshot EventStore history into sorted temporary runs. This preserves the - // global HLC replay order without retaining the complete backlog in memory. - info!("Loading EventStore replay pages..."); - let replay_spool = ReplaySpool::load(eventstore, snapshot.to_sequence_map()).await?; - info!("{} EventStore events spooled.", replay_spool.total_events()); - - info!("Replaying events to actors..."); - // 4. Replay the EventStore events to all listeners (except effects). - // Skip lifecycle infrastructure events. SyncEnded, EffectsEnabled and sync-start events are - // re-published by this sync process; Shutdown belongs to the previous process and would stop - // freshly constructed actors. Replaying these here - // would poison the EventBus deduplication window: the replayed event has the same - // EventId (payload hash) as the one we publish later, causing the later event to be - // silently dropped. This is critical for SyncEnded, if the EvmChainGateway never - // receives it, the gateway stays in BufferUntilLive and all live EVM events are lost. - let replayed = replay_spool.replay(bus).await?; - info!(replayed_events = replayed, "Events replayed."); - - // Loose ends after a crash: - // - // Terminal E3 work that *completed while this node was down* is recovered by the - // historical EVM re-fetch in step 5 below: the terminal on-chain events - // (PlaintextOutputPublished / E3Failed / committee completion) are re-delivered once - // effects are enabled, which re-drives the Sortition release path and frees any tickets - // the node was still holding. So "an E3 finished while we were offline" needs no special - // handling here — it is reconciled by replaying the canonical chain state. - // - // What is intentionally NOT auto-re-driven *here in sync* is this node's *own* in-flight - // request work by replaying the originating request events. Blindly re-publishing the - // originating request event is a no-op: the event bus dedups by EventId (payload hash), so - // the replayed event is dropped. Forcibly minting a fresh EventId to force re-execution is - // unsafe on a value-bearing protocol (it can double-emit or race the canonical chain state) - // and is therefore deliberately left out of the sync path. - // - // Note: this is *not* a global absence of restart recovery. Actors that hold determined, - // idempotent in-flight results re-drive themselves when `EffectsEnabled` is broadcast at the - // end of this sync (e.g. `ThresholdKeyshare::resume_in_flight_work` re-publishes a computed - // keyshare / decryption share). What sync deliberately avoids is replaying *request* events. - // - // Detection of loose ends that cannot be locally re-driven is exposed offline and - // non-destructively via `interfold node validate`, which cross-checks the persisted committee - // slots against terminal events in the log and reports orphaned tickets. See - // `crates/entrypoint/src/validate.rs`. - - // 5. Load the historical evm events to memory from all chains - info!("Loading historical blockchain events..."); - let (addr, rx) = actix_toolbox::mpsc::(256); - bus.publish_without_context(HistoricalEvmSyncStart::new(addr, evm_config.clone()))?; - let historical_evm_events = collect_historical_evm_events(rx, &evm_config).await?; - info!( - "{} historical blockchain events loaded.", - historical_evm_events.len() - ); - // Build the net sync cursor using snapshot timestamps (the original HLC timestamps - // from before the restart). See SyncPlanner::net_sync_cursor for why the re-read EVM - // event timestamps cannot be used. - let net_config = SyncPlanner::net_sync_cursor(&historical_evm_events, &snapshot_net_config); - - // 6. Load the historical libp2p events to memory - info!("Waiting until NetReady..."); - net_ready.await?; - info!("NetReady!"); - info!("Loading historical libp2p events..."); - let events_received = bus.wait_for(EventType::HistoricalNetSyncEventsReceived); - bus.publish_without_context(HistoricalNetSyncStart::new(net_config.clone()))?; - let InterfoldEventData::HistoricalNetSyncEventsReceived(event) = - events_received.await?.into_data() - else { - bail!("failed to get HistoricalNetSyncEventsReceived"); - }; - let historical_net_events = event.events; - info!( - "{} historical libp2p events loaded.", - historical_net_events.len() - ); - - // 7. Sort both the evm and libp2p events together by HLC timestamp - let mut historical = historical_evm_events - .into_iter() - .chain(historical_net_events) - .collect::>(); - - SyncPlanner::sort_by_timestamp(&mut historical); - info!("Historical events sorted."); - - // 8-10. Enable effects, publish canonical history, then enter live mode. Each phase is fenced - // through durable storage and EventBus fanout so aggregate-specific EventStore response order - // cannot move history ahead of EffectsEnabled or SyncEnded ahead of history. - publish_reconciled_history(bus, historical).await?; - // normal live operations - - Ok(()) -} - -async fn publish_reconciled_history( - bus: &BusHandle, - historical: Vec>, -) -> Result<()> { - info!("Enabling effects..."); - bus.publish_without_context(EffectsEnabled::new())?; - bus.flush_event_pipeline().await?; - info!("Effects enabled."); - - info!("Publishing historical events to actors..."); - for event in historical { - bus.naked_dispatch_async(event).await?; - } - bus.flush_event_pipeline().await?; - info!("Historical events published."); - - info!("Publishing SyncEnded event..."); - bus.publish_without_context(SyncEnded::new())?; - bus.flush_event_pipeline().await?; - info!("Sync finished."); - Ok(()) -} - -#[cfg(test)] -async fn replay_eventstore_events( - bus: &BusHandle, - mut events: Vec, -) -> Result { - let total_events = events.len(); - let mut replayed = 0usize; - - // Snapshot metadata can lag the append-only log after a failed snapshot write. Seed from the - // actual replay set before any subscriber can emit follow-up work, otherwise new local events - // may receive timestamps behind durable post-snapshot history. - if let Some(max_ts) = events.iter().map(EventContextAccessors::ts).max() { - bus.seed_clock(max_ts)?; - } - - // EventStoreRouter gathers one query response per aggregate. Those actor responses can arrive - // in any order, so replay must re-establish the global HLC order before stateful subscribers - // observe cross-aggregate dependencies. - events.sort_by_key(|event| event.ts()); - - for event in events { - if SyncPlanner::classify_replay(&event) == ReplayDecision::SkipInfrastructure { - continue; - } - // Await EventBus handling before submitting the next event. `try_send` lets this producer - // outrun the bounded mailbox and aborts startup when it fills; the awaited Actix request - // preserves replay order, yields between events, and reports a closed mailbox. - bus.event_bus().send(EventBusFanout(event)).await??; - replayed += 1; - - if replayed.is_multiple_of(REPLAY_PROGRESS_INTERVAL) { - info!( - replayed_events = replayed, - total_events, "EventStore replay progress" - ); - } - } - // The EventBus acknowledges its own handler before an awaited subscriber fanout finishes. - // A fence queued after the final replay event therefore proves the last downstream handler - // has completed before startup advances to canonical-chain reconciliation. - bus.event_bus().send(EventBusBarrier).await?; - Ok(replayed) -} - -/// Validate or initialize the durable schema marker before runtime actors can write state. -/// -/// A fresh store is stamped synchronously; incompatible or unversioned protocol state halts. The -/// check is idempotent so composition roots may run it before actor construction while [`sync`] -/// retains the same guard for direct callers. -pub async fn preflight_schema_version( - repositories: &Repositories, - aggregate_config: &AggregateConfig, - eventstore: &Recipient>, -) -> Result<()> { - let repo = repositories.schema_version(); - let persisted = repo.read().await?; - let has_existing_state = if persisted.is_none() { - has_schema_governed_kv_state(repositories).await? - || event_logs_have_events(aggregate_config, eventstore).await? - } else { - false - }; - match decide_schema_version(persisted, SCHEMA_VERSION, has_existing_state) { - SchemaVersionDecision::Proceed => Ok(()), - SchemaVersionDecision::WriteCurrent => { - info!("Stamping on-disk schema version {SCHEMA_VERSION}."); - repo.write_sync(&SCHEMA_VERSION).await?; - Ok(()) - } - SchemaVersionDecision::Halt(reason) => { - bail!("Schema version check failed: {reason}"); - } - } -} - -/// Return whether the key/value store contains state whose interpretation requires a schema -/// marker. -/// -/// Wallet setup runs before node startup and atomically writes the encrypted operator and libp2p -/// identities. That complete pair is the only pre-schema key/value state considered fresh. A -/// partial identity, any additional key, or any other non-empty key set remains schema-governed so -/// startup fails closed instead of asserting compatibility with unknown bytes. -pub async fn has_schema_governed_kv_state(repositories: &Repositories) -> Result { - if repositories.store.is_empty().await? { - return Ok(false); - } - - let bootstrap_identity_keys = [StoreKeys::eth_private_key(), StoreKeys::libp2p_keypair()]; - Ok(!repositories - .store - .has_exact_keys(bootstrap_identity_keys) - .await?) -} - -async fn event_logs_have_events( - aggregate_config: &AggregateConfig, - eventstore: &Recipient>, -) -> Result { - let query = aggregate_config - .aggregates() - .into_iter() - .map(|aggregate_id| (aggregate_id, 1)) - .collect(); - let (response, receiver) = actix_toolbox::oneshot::(); - eventstore - .send(EventStoreQueryBy::::new(CorrelationId::new(), query, response).with_limit(1)) - .await - .context("event-store router stopped during schema preflight")?; - Ok(!receiver - .await - .context("event-store query stopped during schema preflight")? - .into_events() - .is_empty()) -} - -pub async fn collect_historical_evm_events( - mut receiver: Receiver, - config: &EvmEventConfig, -) -> Result>> { - let mut collector = HistoricalEvmCollector::new(config); - let progress_interval = Duration::from_secs(30); - - while !collector.is_complete() { - match tokio::time::timeout(progress_interval, receiver.recv()).await { - Ok(Some(mut msg)) => { - let chain_id = msg.chain_id; - if let CollectOutcome::Recorded { - chains_received, - chains_expected, - } = collector.record(&mut msg) - { - info!( - chain_id, - chains_received, chains_expected, "Received historical events from chain" - ); - } - } - Ok(None) => { - let remaining = collector.remaining(); - bail!("historical EVM event channel closed before chains reported: {remaining:?}"); - } - Err(_) => { - // Not a failure — just a progress heartbeat - let remaining = collector.remaining(); - info!( - ?remaining, - "Still waiting for historical events from chains" - ); - continue; - } - } - } - - Ok(collector.into_events()) -} - -#[derive(Message)] -#[rtype("()")] -pub struct Bootstrap; - -#[derive(Message)] -#[rtype("()")] -pub struct SnapshotLoaded { - pub snapshot: SnapshotMeta, -} -impl SnapshotLoaded { - pub fn new(snapshot: SnapshotMeta) -> Self { - Self { snapshot } - } -} - -#[cfg(test)] -mod tests { - use super::{ - collect_historical_evm_events, has_schema_governed_kv_state, preflight_schema_version, - publish_reconciled_history, replay_eventstore_events, - }; - use crate::SyncRepositoryFactory; - use e3_ciphernode_builder::EventSystem; - use e3_data::Repositories; - use e3_events::{ - hlc::{Hlc, HlcTimestamp}, - EffectsEnabled, Event, EventContextAccessors, EventPublisher, EventSubscriber, EventType, - EvmEventConfig, EvmEventConfigChain, GetEvents, HistoricalEvmEventsReceived, - HistoricalEvmSyncStart, InterfoldEvent, InterfoldEventData, StoreKeys, SyncEnded, - TakeEvents, Unsequenced, - }; - use e3_utils::MAILBOX_LIMIT_LARGE; - use std::collections::BTreeMap; - - fn make_historical_evm_sync_start() -> HistoricalEvmSyncStart { - HistoricalEvmSyncStart { - evm_config: EvmEventConfig::new(), - sender: None, - } - } - - fn evm_config(chains: &[u64]) -> EvmEventConfig { - EvmEventConfig::from_config( - chains - .iter() - .map(|chain_id| (*chain_id, EvmEventConfigChain::new(0))) - .collect::>(), - ) - } - - fn historical_batch(chain_id: u64, event_count: usize) -> HistoricalEvmEventsReceived { - let events = (0..event_count) - .map(|index| { - InterfoldEvent::::test_event("historical") - .id(index as u64 + 1) - .build() - }) - .collect(); - HistoricalEvmEventsReceived::new(events, chain_id) - } - - #[actix::test] - async fn schema_preflight_rejects_unversioned_snapshot_state() -> anyhow::Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let store = system.store()?; - store.scope("legacy").write_sync(7_u64).await?; - let repositories = Repositories::from(&store); - let eventstore = system.eventstore_reader()?.seq(); - - let error = - preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) - .await - .unwrap_err(); - - assert!(error.to_string().contains("no schema marker")); - Ok(()) - } - - #[actix::test] - async fn schema_preflight_initializes_store_with_only_bootstrap_identity() -> anyhow::Result<()> - { - let system = EventSystem::new().with_fresh_bus(); - let store = system.store()?; - let private_key = vec![1_u8, 2, 3]; - let network_key = vec![4_u8, 5, 6]; - store - .write_batch_sync([ - (StoreKeys::eth_private_key(), private_key.clone()), - (StoreKeys::libp2p_keypair(), network_key.clone()), - ]) - .await?; - let repositories = Repositories::from(&store); - let eventstore = system.eventstore_reader()?.seq(); - - assert!(!has_schema_governed_kv_state(&repositories).await?); - preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore).await?; - - assert_eq!( - repositories.schema_version().read().await?, - Some(super::SCHEMA_VERSION) - ); - assert_eq!( - store - .scope(StoreKeys::eth_private_key()) - .read::>() - .await?, - Some(private_key) - ); - assert_eq!( - store - .scope(StoreKeys::libp2p_keypair()) - .read::>() - .await?, - Some(network_key) - ); - Ok(()) - } - - #[actix::test] - async fn schema_preflight_rejects_bootstrap_identity_plus_protocol_state() -> anyhow::Result<()> - { - let system = EventSystem::new().with_fresh_bus(); - let store = system.store()?; - store - .write_batch_sync([ - (StoreKeys::eth_private_key(), vec![1_u8]), - (StoreKeys::libp2p_keypair(), vec![2_u8]), - ]) - .await?; - store - .scope(StoreKeys::sortition()) - .write_sync(7_u64) - .await?; - let repositories = Repositories::from(&store); - let eventstore = system.eventstore_reader()?.seq(); - - assert!(has_schema_governed_kv_state(&repositories).await?); - let error = - preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) - .await - .unwrap_err(); - - assert!(error.to_string().contains("no schema marker")); - assert_eq!(repositories.schema_version().read().await?, None); - Ok(()) - } - - #[actix::test] - async fn schema_preflight_rejects_partial_bootstrap_identity() -> anyhow::Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let store = system.store()?; - store - .scope(StoreKeys::eth_private_key()) - .write_sync(vec![1_u8]) - .await?; - let repositories = Repositories::from(&store); - let eventstore = system.eventstore_reader()?.seq(); - - let error = - preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) - .await - .unwrap_err(); - - assert!(error.to_string().contains("no schema marker")); - assert_eq!(repositories.schema_version().read().await?, None); - Ok(()) - } - - #[actix::test] - async fn schema_preflight_rejects_unversioned_event_log() -> anyhow::Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let store = system.store()?; - let repositories = Repositories::from(&store); - let bus = system.handle()?.enable("unversioned-event-log"); - bus.publish_without_context(e3_events::TestEvent::new("legacy", 1))?; - bus.flush_event_pipeline().await?; - let eventstore = system.eventstore_reader()?.seq(); - - let error = - preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) - .await - .unwrap_err(); - - assert!(error.to_string().contains("no schema marker")); - Ok(()) - } - - #[actix::test] - async fn historical_evm_collection_fails_when_any_chain_disconnects() { - let (sender, receiver) = tokio::sync::mpsc::channel(2); - sender.send(historical_batch(1, 2)).await.unwrap(); - drop(sender); - - let error = collect_historical_evm_events(receiver, &evm_config(&[1, 2, 3])) - .await - .unwrap_err(); - - assert_eq!( - error.to_string(), - "historical EVM event channel closed before chains reported: [2, 3]" - ); - } - - #[actix::test] - async fn historical_evm_collection_returns_only_after_every_chain_reports() { - let (sender, receiver) = tokio::sync::mpsc::channel(2); - sender.send(historical_batch(2, 3)).await.unwrap(); - sender.send(historical_batch(1, 2)).await.unwrap(); - - let events = collect_historical_evm_events(receiver, &evm_config(&[1, 2])) - .await - .unwrap(); - - assert_eq!(events.len(), 5); - } - - #[actix::test] - async fn infrastructure_events_are_filtered_during_replay() -> anyhow::Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("test-sync-replay"); - let history = bus.history(); - - let events: Vec = vec![ - InterfoldEvent::::test_event("before") - .id(1) - .seq(1) - .build(), - InterfoldEvent::::test_event("sync") - .data(SyncEnded::new()) - .seq(2) - .build(), - InterfoldEvent::::test_event("fx") - .data(EffectsEnabled::new()) - .seq(3) - .build(), - InterfoldEvent::::test_event("evm") - .data(make_historical_evm_sync_start()) - .seq(4) - .build(), - InterfoldEvent::::test_event("after") - .id(2) - .seq(5) - .build(), - ]; - - let replayed = replay_eventstore_events(&bus, events).await?; - assert_eq!(replayed, 2); - - let received = history.send(TakeEvents::new(2)).await?; - - let event_types: Vec<&'static str> = received - .events - .iter() - .map(|e| match e.get_data() { - InterfoldEventData::TestEvent(_) => "TestEvent", - InterfoldEventData::SyncEnded(_) => "SyncEnded", - InterfoldEventData::EffectsEnabled(_) => "EffectsEnabled", - InterfoldEventData::HistoricalEvmSyncStart(_) => "HistoricalEvmSyncStart", - _ => "other", - }) - .collect(); - - assert_eq!(event_types, vec!["TestEvent", "TestEvent"]); - - let msgs: Vec = received - .events - .iter() - .filter_map(|e| { - if let InterfoldEventData::TestEvent(t) = e.get_data() { - Some(t.msg.clone()) - } else { - None - } - }) - .collect(); - - assert_eq!(msgs, vec!["before", "after"]); - Ok(()) - } - - #[actix::test] - async fn replay_backlog_larger_than_event_bus_mailbox_is_delivered() -> anyhow::Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("test-large-sync-replay"); - let history = bus.history(); - let count = MAILBOX_LIMIT_LARGE * 2; - let events = (0..count) - .map(|i| { - InterfoldEvent::::test_event("replay") - .id(i as u64 + 1) - .seq(i as u64 + 1) - .build() - }) - .collect(); - - let replayed = replay_eventstore_events(&bus, events).await?; - assert_eq!(replayed, count); - - let received = history.send(TakeEvents::new(count)).await?; - assert!(!received.timed_out, "all replay events should be delivered"); - assert_eq!(received.events.len(), count); - Ok(()) - } - - #[actix::test] - async fn replay_restores_global_timestamp_order_across_aggregates() -> anyhow::Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("test-ordered-sync-replay"); - let history = bus.history(); - let events = vec![ - InterfoldEvent::::test_event("third") - .id(3) - .aggregate_id(3) - .ts(30) - .seq(1) - .build(), - InterfoldEvent::::test_event("first") - .id(1) - .aggregate_id(1) - .ts(10) - .seq(1) - .build(), - InterfoldEvent::::test_event("second") - .id(2) - .aggregate_id(2) - .ts(20) - .seq(1) - .build(), - ]; - - replay_eventstore_events(&bus, events).await?; - - let received = history.send(TakeEvents::new(3)).await?; - let timestamps = received - .events - .iter() - .map(|event| event.ts()) - .collect::>(); - assert_eq!(timestamps, vec![10, 20, 30]); - Ok(()) - } - - #[actix::test] - async fn replay_seeds_clock_from_post_snapshot_log_history() -> anyhow::Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let bus = system - .handle()? - .enable_with_hlc(Hlc::new(7).with_clock(|| 1_000)); - let durable = HlcTimestamp::new(5_000, 17, 99); - let events = vec![InterfoldEvent::::test_event("durable") - .id(1) - .ts(durable.to_u128()) - .seq(1) - .build()]; - - replay_eventstore_events(&bus, events).await?; - - assert!(HlcTimestamp::from(bus.ts()?) > durable); - Ok(()) - } - - #[actix::test] - async fn startup_history_is_fenced_between_effects_and_live_mode() -> anyhow::Result<()> { - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("test-startup-history-fences"); - let history = bus.history(); - let historical = vec![ - InterfoldEvent::::test_event("first") - .id(1) - .ts(10) - .build(), - InterfoldEvent::::test_event("second") - .id(2) - .ts(20) - .build(), - ]; - - publish_reconciled_history(&bus, historical).await?; - - let received = history.send(GetEvents::new()).await?; - let types = received - .iter() - .map(|event| event.event_type()) - .collect::>(); - assert_eq!( - types, - ["EffectsEnabled", "TestEvent", "TestEvent", "SyncEnded"] - ); - Ok(()) - } - - /// Verify that `run_once::` correctly gates event subscriptions. - /// - /// Simulates the sync flow: - /// 1. An event is published BEFORE EffectsEnabled (should be dropped — nobody listening) - /// 2. EffectsEnabled is published (triggers subscription) - /// 3. The same event is published AFTER EffectsEnabled (should be received) - /// - /// This is the pattern used by Sortition (E3Requested), CommitteeFinalizer - /// (CommitteeRequested), Multithread (ComputeRequest), and the sol writers. - #[actix::test] - async fn effects_enabled_gates_event_subscriptions() -> anyhow::Result<()> { - use std::sync::{ - atomic::{AtomicU32, Ordering}, - Arc, - }; - - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("test-effects-gating"); - - let receive_count = Arc::new(AtomicU32::new(0)); - - // Set up a gated subscription: only subscribe to TestEvent after EffectsEnabled - let counter = receive_count.clone(); - let runner = e3_events::run_once::({ - let bus = bus.clone(); - move |_| { - // Create a simple actor that counts received TestEvents - use actix::{Actor, Context, Handler}; - - struct Counter(Arc); - impl Actor for Counter { - type Context = Context; - } - impl Handler for Counter { - type Result = (); - fn handle( - &mut self, - msg: InterfoldEvent, - _: &mut Self::Context, - ) -> Self::Result { - if matches!(msg.get_data(), InterfoldEventData::TestEvent(_)) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - } - - let addr = Counter(counter).start(); - bus.subscribe(EventType::TestEvent, addr.recipient()); - Ok(()) - } - }); - bus.subscribe(EventType::EffectsEnabled, runner.recipient()); - - // 1. Publish a TestEvent BEFORE EffectsEnabled — should NOT be received - bus.event_bus().try_send( - InterfoldEvent::::test_event("before-effects") - .id(1) - .seq(1) - .build(), - )?; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - assert_eq!( - receive_count.load(Ordering::SeqCst), - 0, - "Event before EffectsEnabled should not be received" - ); - - // 2. Publish EffectsEnabled — triggers the subscription - bus.publish_without_context(EffectsEnabled::new())?; - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - // 3. Publish a TestEvent AFTER EffectsEnabled — should be received - bus.event_bus().try_send( - InterfoldEvent::::test_event("after-effects") - .id(2) - .seq(2) - .build(), - )?; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - assert_eq!( - receive_count.load(Ordering::SeqCst), - 1, - "Event after EffectsEnabled should be received exactly once" - ); - - Ok(()) - } - - /// Verify that ungated (immediate) subscriptions receive events both - /// before and after EffectsEnabled. - /// - /// This mirrors how Sortition subscribes to state-building events - /// (CiphernodeAdded, E3Failed, etc.) immediately, while gating - /// E3Requested behind EffectsEnabled. The immediate subscriptions - /// must work during EventStore replay (before EffectsEnabled). - #[actix::test] - async fn immediate_subscriptions_receive_before_effects_enabled() -> anyhow::Result<()> { - use std::sync::{ - atomic::{AtomicU32, Ordering}, - Arc, - }; - - let system = EventSystem::new().with_fresh_bus(); - let bus = system.handle()?.enable("test-immediate-sub"); - - let immediate_count = Arc::new(AtomicU32::new(0)); - let gated_count = Arc::new(AtomicU32::new(0)); - - // Helper actor that counts TestEvents - use actix::{Actor, Context, Handler}; - - struct Counter(Arc); - impl Actor for Counter { - type Context = Context; - } - impl Handler for Counter { - type Result = (); - fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { - if matches!(msg.get_data(), InterfoldEventData::TestEvent(_)) { - self.0.fetch_add(1, Ordering::SeqCst); - } - } - } - - // Immediate subscription — receives all events, including before EffectsEnabled - let immediate_actor = Counter(immediate_count.clone()).start(); - bus.subscribe(EventType::TestEvent, immediate_actor.recipient()); - - // Gated subscription — only receives after EffectsEnabled - let gated_counter = gated_count.clone(); - let runner = e3_events::run_once::({ - let bus = bus.clone(); - move |_| { - let addr = Counter(gated_counter).start(); - bus.subscribe(EventType::TestEvent, addr.recipient()); - Ok(()) - } - }); - bus.subscribe(EventType::EffectsEnabled, runner.recipient()); - - // 1. Publish event BEFORE EffectsEnabled - bus.event_bus().try_send( - InterfoldEvent::::test_event("during-replay") - .id(1) - .seq(1) - .build(), - )?; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - assert_eq!( - immediate_count.load(Ordering::SeqCst), - 1, - "Immediate subscription should receive events before EffectsEnabled" - ); - assert_eq!( - gated_count.load(Ordering::SeqCst), - 0, - "Gated subscription should NOT receive events before EffectsEnabled" - ); - - // 2. Publish EffectsEnabled - bus.publish_without_context(EffectsEnabled::new())?; - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - // 3. Publish event AFTER EffectsEnabled - bus.event_bus().try_send( - InterfoldEvent::::test_event("after-effects") - .id(2) - .seq(2) - .build(), - )?; - - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - assert_eq!( - immediate_count.load(Ordering::SeqCst), - 2, - "Immediate subscription should receive events after EffectsEnabled too" - ); - assert_eq!( - gated_count.load(Ordering::SeqCst), - 1, - "Gated subscription should receive events after EffectsEnabled" - ); - - Ok(()) - } -} diff --git a/crates/sync/src/domain/mod.rs b/crates/sync/src/domain.rs similarity index 73% rename from crates/sync/src/domain/mod.rs rename to crates/sync/src/domain.rs index 868d3509f0..e5a13c5222 100644 --- a/crates/sync/src/domain/mod.rs +++ b/crates/sync/src/domain.rs @@ -4,9 +4,15 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +//! Compatibility view of pure planning and state stored in the sync capability. + +#[path = "sync/collect_history.rs"] mod historical_evm_collector; +#[path = "sync/schema_version.rs"] mod schema_version; +#[path = "sync/state.rs"] mod snapshot_meta; +#[path = "sync/workflow.rs"] mod sync_planner; pub use schema_version::{decide_schema_version, SchemaVersionDecision, SCHEMA_VERSION}; diff --git a/crates/sync/src/lib.rs b/crates/sync/src/lib.rs index 47b82243fe..2d07d366fc 100644 --- a/crates/sync/src/lib.rs +++ b/crates/sync/src/lib.rs @@ -4,11 +4,11 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -mod actors; mod domain; mod replay_spool; mod repo; +mod runtime; -pub use actors::*; pub use domain::*; pub use repo::*; +pub use runtime::*; diff --git a/crates/sync/src/actors/mod.rs b/crates/sync/src/runtime.rs similarity index 72% rename from crates/sync/src/actors/mod.rs rename to crates/sync/src/runtime.rs index 07b21af2e0..4cb4f22f7c 100644 --- a/crates/sync/src/actors/mod.rs +++ b/crates/sync/src/runtime.rs @@ -4,6 +4,9 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +//! Compatibility view of the effectful sync service. + +#[path = "sync/service.rs"] mod sync; pub use sync::*; diff --git a/crates/sync/src/domain/historical_evm_collector.rs b/crates/sync/src/sync/collect_history.rs similarity index 100% rename from crates/sync/src/domain/historical_evm_collector.rs rename to crates/sync/src/sync/collect_history.rs diff --git a/crates/sync/src/sync/history.rs b/crates/sync/src/sync/history.rs new file mode 100644 index 0000000000..ff13616f40 --- /dev/null +++ b/crates/sync/src/sync/history.rs @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Bounded collection of per-chain historical EVM results. + +use super::*; + +pub async fn collect_historical_evm_events( + mut receiver: Receiver, + config: &EvmEventConfig, +) -> Result>> { + let mut collector = HistoricalEvmCollector::new(config); + let progress_interval = Duration::from_secs(30); + + while !collector.is_complete() { + match tokio::time::timeout(progress_interval, receiver.recv()).await { + Ok(Some(mut msg)) => { + let chain_id = msg.chain_id; + if let CollectOutcome::Recorded { + chains_received, + chains_expected, + } = collector.record(&mut msg) + { + info!( + chain_id, + chains_received, chains_expected, "Received historical events from chain" + ); + } + } + Ok(None) => { + let remaining = collector.remaining(); + bail!("historical EVM event channel closed before chains reported: {remaining:?}"); + } + Err(_) => { + info!( + remaining = ?collector.remaining(), + "Still waiting for historical events from chains" + ); + } + } + } + + Ok(collector.into_events()) +} diff --git a/crates/sync/src/sync/preflight.rs b/crates/sync/src/sync/preflight.rs new file mode 100644 index 0000000000..1b9b246803 --- /dev/null +++ b/crates/sync/src/sync/preflight.rs @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Durable schema admission before runtime state is loaded. + +use super::*; + +/// Validate or initialize the durable schema marker before runtime actors can write state. +pub async fn preflight_schema_version( + repositories: &Repositories, + aggregate_config: &AggregateConfig, + eventstore: &Recipient>, +) -> Result<()> { + let repo = repositories.schema_version(); + let persisted = repo.read().await?; + let has_existing_state = if persisted.is_none() { + has_schema_governed_kv_state(repositories).await? + || event_logs_have_events(aggregate_config, eventstore).await? + } else { + false + }; + match decide_schema_version(persisted, SCHEMA_VERSION, has_existing_state) { + SchemaVersionDecision::Proceed => Ok(()), + SchemaVersionDecision::WriteCurrent => { + info!("Stamping on-disk schema version {SCHEMA_VERSION}."); + repo.write_sync(&SCHEMA_VERSION).await?; + Ok(()) + } + SchemaVersionDecision::Halt(reason) => { + bail!("Schema version check failed: {reason}"); + } + } +} + +/// Return whether the key/value store contains state whose interpretation requires a schema +/// marker. The complete operator/libp2p bootstrap identity pair is the only fresh exception. +pub async fn has_schema_governed_kv_state(repositories: &Repositories) -> Result { + if repositories.store.is_empty().await? { + return Ok(false); + } + + let bootstrap_identity_keys = [StoreKeys::eth_private_key(), StoreKeys::libp2p_keypair()]; + Ok(!repositories + .store + .has_exact_keys(bootstrap_identity_keys) + .await?) +} + +async fn event_logs_have_events( + aggregate_config: &AggregateConfig, + eventstore: &Recipient>, +) -> Result { + let query = aggregate_config + .aggregates() + .into_iter() + .map(|aggregate_id| (aggregate_id, 1)) + .collect(); + let (response, receiver) = actix_toolbox::oneshot::(); + eventstore + .send(EventStoreQueryBy::::new(CorrelationId::new(), query, response).with_limit(1)) + .await + .context("event-store router stopped during schema preflight")?; + Ok(!receiver + .await + .context("event-store query stopped during schema preflight")? + .into_events() + .is_empty()) +} diff --git a/crates/sync/src/domain/schema_version.rs b/crates/sync/src/sync/schema_version.rs similarity index 100% rename from crates/sync/src/domain/schema_version.rs rename to crates/sync/src/sync/schema_version.rs diff --git a/crates/sync/src/sync/service.rs b/crates/sync/src/sync/service.rs new file mode 100644 index 0000000000..8d683d8fe0 --- /dev/null +++ b/crates/sync/src/sync/service.rs @@ -0,0 +1,255 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +#[cfg(test)] +use crate::domain::ReplayDecision; +use crate::domain::{ + decide_schema_version, CollectOutcome, HistoricalEvmCollector, SchemaVersionDecision, + SnapshotMeta, SyncPlanner, SCHEMA_VERSION, +}; +use crate::replay_spool::ReplaySpool; +use crate::SyncRepositoryFactory; +use actix::{Message, Recipient}; +use anyhow::{bail, Context, Result}; +use e3_data::Repositories; +use e3_events::{ + AggregateConfig, BusHandle, CorrelationId, EffectsEnabled, Event, EventPublisher, + EventStoreQueryBy, EventStoreQueryResponse, EventSubscriber, EventType, EvmEventConfig, + HistoricalEvmEventsReceived, HistoricalEvmSyncStart, HistoricalNetSyncStart, InterfoldEvent, + InterfoldEventData, SeqAgg, StoreKeys, SyncEnded, Unsequenced, +}; +#[cfg(test)] +use e3_events::{EventBusBarrier, EventBusFanout, EventContextAccessors}; +use e3_utils::actix::channel as actix_toolbox; +use std::time::Duration; +use tokio::sync::mpsc::Receiver; +use tracing::info; + +#[cfg(test)] +const REPLAY_PROGRESS_INTERVAL: usize = 10_000; + +pub async fn sync( + bus: &BusHandle, + default_config: &EvmEventConfig, + repositories: &Repositories, + aggregate_config: &AggregateConfig, + eventstore: &Recipient>, +) -> Result<()> { + // 0. start listening early for net ready + let net_ready = bus.wait_for(EventType::NetReady); + + // 0b. Verify the on-disk schema version is compatible with this binary + // before touching any persisted state, so an incompatible upgrade or + // downgrade halts loudly instead of silently loading garbage (H19/H20). + preflight_schema_version(repositories, aggregate_config, eventstore).await?; + + // 1. Load snapsshot metadata + info!("Loading snapshot metadata..."); + let snapshot = + SnapshotMeta::read_from_disk(aggregate_config.aggregates(), default_config, repositories) + .await?; + info!( + "Snapshot metadata loaded for {} aggregates.", + snapshot.aggregates().len() + ); + + // 1b. Restore the HLC ordering floor from the highest persisted aggregate + // timestamp so events created after this restart remain strictly after + // durable history, including its logical counter, even if wall time moved backwards. + if let Some(max_ts) = snapshot.to_net_config().values().copied().max() { + bus.seed_clock(max_ts)?; + } + + // 2. Determine the evm blocks to read from based on the SnapshotMeta + let evm_config = snapshot.to_evm_config(); + let snapshot_net_config = snapshot.to_net_config(); + + // 3. Page post-snapshot EventStore history into sorted temporary runs. This preserves the + // global HLC replay order without retaining the complete backlog in memory. + info!("Loading EventStore replay pages..."); + let replay_spool = ReplaySpool::load(eventstore, snapshot.to_sequence_map()).await?; + info!("{} EventStore events spooled.", replay_spool.total_events()); + + info!("Replaying events to actors..."); + // 4. Replay the EventStore events to all listeners (except effects). + // Skip lifecycle infrastructure events. SyncEnded, EffectsEnabled and sync-start events are + // re-published by this sync process; Shutdown belongs to the previous process and would stop + // freshly constructed actors. Replaying these here + // would poison the EventBus deduplication window: the replayed event has the same + // EventId (payload hash) as the one we publish later, causing the later event to be + // silently dropped. This is critical for SyncEnded, if the EvmChainGateway never + // receives it, the gateway stays in BufferUntilLive and all live EVM events are lost. + let replayed = replay_spool.replay(bus).await?; + info!(replayed_events = replayed, "Events replayed."); + + // Loose ends after a crash: + // + // Terminal E3 work that *completed while this node was down* is recovered by the + // historical EVM re-fetch in step 5 below: the terminal on-chain events + // (PlaintextOutputPublished / E3Failed / committee completion) are re-delivered once + // effects are enabled, which re-drives the Sortition release path and frees any tickets + // the node was still holding. So "an E3 finished while we were offline" needs no special + // handling here — it is reconciled by replaying the canonical chain state. + // + // What is intentionally NOT auto-re-driven *here in sync* is this node's *own* in-flight + // request work by replaying the originating request events. Blindly re-publishing the + // originating request event is a no-op: the event bus dedups by EventId (payload hash), so + // the replayed event is dropped. Forcibly minting a fresh EventId to force re-execution is + // unsafe on a value-bearing protocol (it can double-emit or race the canonical chain state) + // and is therefore deliberately left out of the sync path. + // + // Note: this is *not* a global absence of restart recovery. Actors that hold determined, + // idempotent in-flight results re-drive themselves when `EffectsEnabled` is broadcast at the + // end of this sync (e.g. `ThresholdKeyshare::resume_in_flight_work` re-publishes a computed + // keyshare / decryption share). What sync deliberately avoids is replaying *request* events. + // + // Detection of loose ends that cannot be locally re-driven is exposed offline and + // non-destructively via `interfold node validate`, which cross-checks the persisted committee + // slots against terminal events in the log and reports orphaned tickets. See + // `crates/entrypoint/src/validate.rs`. + + // 5. Load the historical evm events to memory from all chains + info!("Loading historical blockchain events..."); + let (addr, rx) = actix_toolbox::mpsc::(256); + bus.publish_without_context(HistoricalEvmSyncStart::new(addr, evm_config.clone()))?; + let historical_evm_events = collect_historical_evm_events(rx, &evm_config).await?; + info!( + "{} historical blockchain events loaded.", + historical_evm_events.len() + ); + // Build the net sync cursor using snapshot timestamps (the original HLC timestamps + // from before the restart). See SyncPlanner::net_sync_cursor for why the re-read EVM + // event timestamps cannot be used. + let net_config = SyncPlanner::net_sync_cursor(&historical_evm_events, &snapshot_net_config); + + // 6. Load the historical libp2p events to memory + info!("Waiting until NetReady..."); + net_ready.await?; + info!("NetReady!"); + info!("Loading historical libp2p events..."); + let events_received = bus.wait_for(EventType::HistoricalNetSyncEventsReceived); + bus.publish_without_context(HistoricalNetSyncStart::new(net_config.clone()))?; + let InterfoldEventData::HistoricalNetSyncEventsReceived(event) = + events_received.await?.into_data() + else { + bail!("failed to get HistoricalNetSyncEventsReceived"); + }; + let historical_net_events = event.events; + info!( + "{} historical libp2p events loaded.", + historical_net_events.len() + ); + + // 7. Sort both the evm and libp2p events together by HLC timestamp + let mut historical = historical_evm_events + .into_iter() + .chain(historical_net_events) + .collect::>(); + + SyncPlanner::sort_by_timestamp(&mut historical); + info!("Historical events sorted."); + + // 8-10. Enable effects, publish canonical history, then enter live mode. Each phase is fenced + // through durable storage and EventBus fanout so aggregate-specific EventStore response order + // cannot move history ahead of EffectsEnabled or SyncEnded ahead of history. + publish_reconciled_history(bus, historical).await?; + // normal live operations + + Ok(()) +} + +async fn publish_reconciled_history( + bus: &BusHandle, + historical: Vec>, +) -> Result<()> { + info!("Enabling effects..."); + bus.publish_without_context(EffectsEnabled::new())?; + bus.flush_event_pipeline().await?; + info!("Effects enabled."); + + info!("Publishing historical events to actors..."); + for event in historical { + bus.naked_dispatch_async(event).await?; + } + bus.flush_event_pipeline().await?; + info!("Historical events published."); + + info!("Publishing SyncEnded event..."); + bus.publish_without_context(SyncEnded::new())?; + bus.flush_event_pipeline().await?; + info!("Sync finished."); + Ok(()) +} + +#[cfg(test)] +async fn replay_eventstore_events( + bus: &BusHandle, + mut events: Vec, +) -> Result { + let total_events = events.len(); + let mut replayed = 0usize; + + // Snapshot metadata can lag the append-only log after a failed snapshot write. Seed from the + // actual replay set before any subscriber can emit follow-up work, otherwise new local events + // may receive timestamps behind durable post-snapshot history. + if let Some(max_ts) = events.iter().map(EventContextAccessors::ts).max() { + bus.seed_clock(max_ts)?; + } + + // EventStoreRouter gathers one query response per aggregate. Those actor responses can arrive + // in any order, so replay must re-establish the global HLC order before stateful subscribers + // observe cross-aggregate dependencies. + events.sort_by_key(|event| event.ts()); + + for event in events { + if SyncPlanner::classify_replay(&event) == ReplayDecision::SkipInfrastructure { + continue; + } + // Await EventBus handling before submitting the next event. `try_send` lets this producer + // outrun the bounded mailbox and aborts startup when it fills; the awaited Actix request + // preserves replay order, yields between events, and reports a closed mailbox. + bus.event_bus().send(EventBusFanout(event)).await??; + replayed += 1; + + if replayed.is_multiple_of(REPLAY_PROGRESS_INTERVAL) { + info!( + replayed_events = replayed, + total_events, "EventStore replay progress" + ); + } + } + // The EventBus acknowledges its own handler before an awaited subscriber fanout finishes. + // A fence queued after the final replay event therefore proves the last downstream handler + // has completed before startup advances to canonical-chain reconciliation. + bus.event_bus().send(EventBusBarrier).await?; + Ok(replayed) +} + +#[path = "history.rs"] +mod historical; +mod preflight; + +pub use historical::collect_historical_evm_events; +pub use preflight::{has_schema_governed_kv_state, preflight_schema_version}; + +#[derive(Message)] +#[rtype("()")] +pub struct Bootstrap; + +#[derive(Message)] +#[rtype("()")] +pub struct SnapshotLoaded { + pub snapshot: SnapshotMeta, +} +impl SnapshotLoaded { + pub fn new(snapshot: SnapshotMeta) -> Self { + Self { snapshot } + } +} + +#[cfg(test)] +#[path = "tests/mod.rs"] +mod tests; diff --git a/crates/sync/src/domain/snapshot_meta.rs b/crates/sync/src/sync/state.rs similarity index 100% rename from crates/sync/src/domain/snapshot_meta.rs rename to crates/sync/src/sync/state.rs diff --git a/crates/sync/src/sync/tests/gates.rs b/crates/sync/src/sync/tests/gates.rs new file mode 100644 index 0000000000..ce79dcab0c --- /dev/null +++ b/crates/sync/src/sync/tests/gates.rs @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn startup_history_is_fenced_between_effects_and_live_mode() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-startup-history-fences"); + let history = bus.history(); + let historical = vec![ + InterfoldEvent::::test_event("first") + .id(1) + .ts(10) + .build(), + InterfoldEvent::::test_event("second") + .id(2) + .ts(20) + .build(), + ]; + + publish_reconciled_history(&bus, historical).await?; + + let received = history.send(GetEvents::new()).await?; + let types = received + .iter() + .map(|event| event.event_type()) + .collect::>(); + assert_eq!( + types, + ["EffectsEnabled", "TestEvent", "TestEvent", "SyncEnded"] + ); + Ok(()) +} + +/// Verify that `run_once::` correctly gates event subscriptions. +/// +/// Simulates the sync flow: +/// 1. An event is published BEFORE EffectsEnabled (should be dropped — nobody listening) +/// 2. EffectsEnabled is published (triggers subscription) +/// 3. The same event is published AFTER EffectsEnabled (should be received) +/// +/// This is the pattern used by Sortition (E3Requested), CommitteeFinalizer +/// (CommitteeRequested), Multithread (ComputeRequest), and the sol writers. +#[actix::test] +async fn effects_enabled_gates_event_subscriptions() -> anyhow::Result<()> { + use std::sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }; + + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-effects-gating"); + + let receive_count = Arc::new(AtomicU32::new(0)); + + // Set up a gated subscription: only subscribe to TestEvent after EffectsEnabled + let counter = receive_count.clone(); + let runner = e3_events::run_once::({ + let bus = bus.clone(); + move |_| { + // Create a simple actor that counts received TestEvents + use actix::{Actor, Context, Handler}; + + struct Counter(Arc); + impl Actor for Counter { + type Context = Context; + } + impl Handler for Counter { + type Result = (); + fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { + if matches!(msg.get_data(), InterfoldEventData::TestEvent(_)) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + } + + let addr = Counter(counter).start(); + bus.subscribe(EventType::TestEvent, addr.recipient()); + Ok(()) + } + }); + bus.subscribe(EventType::EffectsEnabled, runner.recipient()); + + // 1. Publish a TestEvent BEFORE EffectsEnabled — should NOT be received + bus.event_bus().try_send( + InterfoldEvent::::test_event("before-effects") + .id(1) + .seq(1) + .build(), + )?; + + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert_eq!( + receive_count.load(Ordering::SeqCst), + 0, + "Event before EffectsEnabled should not be received" + ); + + // 2. Publish EffectsEnabled — triggers the subscription + bus.publish_without_context(EffectsEnabled::new())?; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + // 3. Publish a TestEvent AFTER EffectsEnabled — should be received + bus.event_bus().try_send( + InterfoldEvent::::test_event("after-effects") + .id(2) + .seq(2) + .build(), + )?; + + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert_eq!( + receive_count.load(Ordering::SeqCst), + 1, + "Event after EffectsEnabled should be received exactly once" + ); + + Ok(()) +} + +/// Verify that ungated (immediate) subscriptions receive events both +/// before and after EffectsEnabled. +/// +/// This mirrors how Sortition subscribes to state-building events +/// (CiphernodeAdded, E3Failed, etc.) immediately, while gating +/// E3Requested behind EffectsEnabled. The immediate subscriptions +/// must work during EventStore replay (before EffectsEnabled). +#[actix::test] +async fn immediate_subscriptions_receive_before_effects_enabled() -> anyhow::Result<()> { + use std::sync::{ + atomic::{AtomicU32, Ordering}, + Arc, + }; + + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-immediate-sub"); + + let immediate_count = Arc::new(AtomicU32::new(0)); + let gated_count = Arc::new(AtomicU32::new(0)); + + // Helper actor that counts TestEvents + use actix::{Actor, Context, Handler}; + + struct Counter(Arc); + impl Actor for Counter { + type Context = Context; + } + impl Handler for Counter { + type Result = (); + fn handle(&mut self, msg: InterfoldEvent, _: &mut Self::Context) -> Self::Result { + if matches!(msg.get_data(), InterfoldEventData::TestEvent(_)) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + } + + // Immediate subscription — receives all events, including before EffectsEnabled + let immediate_actor = Counter(immediate_count.clone()).start(); + bus.subscribe(EventType::TestEvent, immediate_actor.recipient()); + + // Gated subscription — only receives after EffectsEnabled + let gated_counter = gated_count.clone(); + let runner = e3_events::run_once::({ + let bus = bus.clone(); + move |_| { + let addr = Counter(gated_counter).start(); + bus.subscribe(EventType::TestEvent, addr.recipient()); + Ok(()) + } + }); + bus.subscribe(EventType::EffectsEnabled, runner.recipient()); + + // 1. Publish event BEFORE EffectsEnabled + bus.event_bus().try_send( + InterfoldEvent::::test_event("during-replay") + .id(1) + .seq(1) + .build(), + )?; + + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert_eq!( + immediate_count.load(Ordering::SeqCst), + 1, + "Immediate subscription should receive events before EffectsEnabled" + ); + assert_eq!( + gated_count.load(Ordering::SeqCst), + 0, + "Gated subscription should NOT receive events before EffectsEnabled" + ); + + // 2. Publish EffectsEnabled + bus.publish_without_context(EffectsEnabled::new())?; + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + // 3. Publish event AFTER EffectsEnabled + bus.event_bus().try_send( + InterfoldEvent::::test_event("after-effects") + .id(2) + .seq(2) + .build(), + )?; + + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert_eq!( + immediate_count.load(Ordering::SeqCst), + 2, + "Immediate subscription should receive events after EffectsEnabled too" + ); + assert_eq!( + gated_count.load(Ordering::SeqCst), + 1, + "Gated subscription should receive events after EffectsEnabled" + ); + + Ok(()) +} diff --git a/crates/sync/src/sync/tests/history.rs b/crates/sync/src/sync/tests/history.rs new file mode 100644 index 0000000000..f9f3a19b5e --- /dev/null +++ b/crates/sync/src/sync/tests/history.rs @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn historical_evm_collection_fails_when_any_chain_disconnects() { + let (sender, receiver) = tokio::sync::mpsc::channel(2); + sender.send(historical_batch(1, 2)).await.unwrap(); + drop(sender); + + let error = collect_historical_evm_events(receiver, &evm_config(&[1, 2, 3])) + .await + .unwrap_err(); + + assert_eq!( + error.to_string(), + "historical EVM event channel closed before chains reported: [2, 3]" + ); +} + +#[actix::test] +async fn historical_evm_collection_returns_only_after_every_chain_reports() { + let (sender, receiver) = tokio::sync::mpsc::channel(2); + sender.send(historical_batch(2, 3)).await.unwrap(); + sender.send(historical_batch(1, 2)).await.unwrap(); + + let events = collect_historical_evm_events(receiver, &evm_config(&[1, 2])) + .await + .unwrap(); + + assert_eq!(events.len(), 5); +} diff --git a/crates/sync/src/sync/tests/mod.rs b/crates/sync/src/sync/tests/mod.rs new file mode 100644 index 0000000000..195389c350 --- /dev/null +++ b/crates/sync/src/sync/tests/mod.rs @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::{ + collect_historical_evm_events, has_schema_governed_kv_state, preflight_schema_version, + publish_reconciled_history, replay_eventstore_events, +}; +use crate::{SyncRepositoryFactory, SCHEMA_VERSION}; +use e3_ciphernode_builder::EventSystem; +use e3_data::Repositories; +use e3_events::{ + hlc::{Hlc, HlcTimestamp}, + EffectsEnabled, Event, EventContextAccessors, EventPublisher, EventSubscriber, EventType, + EvmEventConfig, EvmEventConfigChain, GetEvents, HistoricalEvmEventsReceived, + HistoricalEvmSyncStart, InterfoldEvent, InterfoldEventData, StoreKeys, SyncEnded, TakeEvents, + Unsequenced, +}; +use e3_utils::MAILBOX_LIMIT_LARGE; +use std::collections::BTreeMap; + +fn make_historical_evm_sync_start() -> HistoricalEvmSyncStart { + HistoricalEvmSyncStart { + evm_config: EvmEventConfig::new(), + sender: None, + } +} + +fn evm_config(chains: &[u64]) -> EvmEventConfig { + EvmEventConfig::from_config( + chains + .iter() + .map(|chain_id| (*chain_id, EvmEventConfigChain::new(0))) + .collect::>(), + ) +} + +fn historical_batch(chain_id: u64, event_count: usize) -> HistoricalEvmEventsReceived { + let events = (0..event_count) + .map(|index| { + InterfoldEvent::::test_event("historical") + .id(index as u64 + 1) + .build() + }) + .collect(); + HistoricalEvmEventsReceived::new(events, chain_id) +} + +mod gates; +#[path = "history.rs"] +mod historical; +mod replay; +mod schema; diff --git a/crates/sync/src/sync/tests/replay.rs b/crates/sync/src/sync/tests/replay.rs new file mode 100644 index 0000000000..9497a6f136 --- /dev/null +++ b/crates/sync/src/sync/tests/replay.rs @@ -0,0 +1,152 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn infrastructure_events_are_filtered_during_replay() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-sync-replay"); + let history = bus.history(); + + let events: Vec = vec![ + InterfoldEvent::::test_event("before") + .id(1) + .seq(1) + .build(), + InterfoldEvent::::test_event("sync") + .data(SyncEnded::new()) + .seq(2) + .build(), + InterfoldEvent::::test_event("fx") + .data(EffectsEnabled::new()) + .seq(3) + .build(), + InterfoldEvent::::test_event("evm") + .data(make_historical_evm_sync_start()) + .seq(4) + .build(), + InterfoldEvent::::test_event("after") + .id(2) + .seq(5) + .build(), + ]; + + let replayed = replay_eventstore_events(&bus, events).await?; + assert_eq!(replayed, 2); + + let received = history.send(TakeEvents::new(2)).await?; + + let event_types: Vec<&'static str> = received + .events + .iter() + .map(|e| match e.get_data() { + InterfoldEventData::TestEvent(_) => "TestEvent", + InterfoldEventData::SyncEnded(_) => "SyncEnded", + InterfoldEventData::EffectsEnabled(_) => "EffectsEnabled", + InterfoldEventData::HistoricalEvmSyncStart(_) => "HistoricalEvmSyncStart", + _ => "other", + }) + .collect(); + + assert_eq!(event_types, vec!["TestEvent", "TestEvent"]); + + let msgs: Vec = received + .events + .iter() + .filter_map(|e| { + if let InterfoldEventData::TestEvent(t) = e.get_data() { + Some(t.msg.clone()) + } else { + None + } + }) + .collect(); + + assert_eq!(msgs, vec!["before", "after"]); + Ok(()) +} + +#[actix::test] +async fn replay_backlog_larger_than_event_bus_mailbox_is_delivered() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-large-sync-replay"); + let history = bus.history(); + let count = MAILBOX_LIMIT_LARGE * 2; + let events = (0..count) + .map(|i| { + InterfoldEvent::::test_event("replay") + .id(i as u64 + 1) + .seq(i as u64 + 1) + .build() + }) + .collect(); + + let replayed = replay_eventstore_events(&bus, events).await?; + assert_eq!(replayed, count); + + let received = history.send(TakeEvents::new(count)).await?; + assert!(!received.timed_out, "all replay events should be delivered"); + assert_eq!(received.events.len(), count); + Ok(()) +} + +#[actix::test] +async fn replay_restores_global_timestamp_order_across_aggregates() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("test-ordered-sync-replay"); + let history = bus.history(); + let events = vec![ + InterfoldEvent::::test_event("third") + .id(3) + .aggregate_id(3) + .ts(30) + .seq(1) + .build(), + InterfoldEvent::::test_event("first") + .id(1) + .aggregate_id(1) + .ts(10) + .seq(1) + .build(), + InterfoldEvent::::test_event("second") + .id(2) + .aggregate_id(2) + .ts(20) + .seq(1) + .build(), + ]; + + replay_eventstore_events(&bus, events).await?; + + let received = history.send(TakeEvents::new(3)).await?; + let timestamps = received + .events + .iter() + .map(|event| event.ts()) + .collect::>(); + assert_eq!(timestamps, vec![10, 20, 30]); + Ok(()) +} + +#[actix::test] +async fn replay_seeds_clock_from_post_snapshot_log_history() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system + .handle()? + .enable_with_hlc(Hlc::new(7).with_clock(|| 1_000)); + let durable = HlcTimestamp::new(5_000, 17, 99); + let events = vec![InterfoldEvent::::test_event("durable") + .id(1) + .ts(durable.to_u128()) + .seq(1) + .build()]; + + replay_eventstore_events(&bus, events).await?; + + assert!(HlcTimestamp::from(bus.ts()?) > durable); + Ok(()) +} diff --git a/crates/sync/src/sync/tests/schema.rs b/crates/sync/src/sync/tests/schema.rs new file mode 100644 index 0000000000..917395e478 --- /dev/null +++ b/crates/sync/src/sync/tests/schema.rs @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[actix::test] +async fn schema_preflight_rejects_unversioned_snapshot_state() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let store = system.store()?; + store.scope("legacy").write_sync(7_u64).await?; + let repositories = Repositories::from(&store); + let eventstore = system.eventstore_reader()?.seq(); + + let error = preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) + .await + .unwrap_err(); + + assert!(error.to_string().contains("no schema marker")); + Ok(()) +} + +#[actix::test] +async fn schema_preflight_initializes_store_with_only_bootstrap_identity() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let store = system.store()?; + let private_key = vec![1_u8, 2, 3]; + let network_key = vec![4_u8, 5, 6]; + store + .write_batch_sync([ + (StoreKeys::eth_private_key(), private_key.clone()), + (StoreKeys::libp2p_keypair(), network_key.clone()), + ]) + .await?; + let repositories = Repositories::from(&store); + let eventstore = system.eventstore_reader()?.seq(); + + assert!(!has_schema_governed_kv_state(&repositories).await?); + preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore).await?; + + assert_eq!( + repositories.schema_version().read().await?, + Some(super::SCHEMA_VERSION) + ); + assert_eq!( + store + .scope(StoreKeys::eth_private_key()) + .read::>() + .await?, + Some(private_key) + ); + assert_eq!( + store + .scope(StoreKeys::libp2p_keypair()) + .read::>() + .await?, + Some(network_key) + ); + Ok(()) +} + +#[actix::test] +async fn schema_preflight_rejects_bootstrap_identity_plus_protocol_state() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let store = system.store()?; + store + .write_batch_sync([ + (StoreKeys::eth_private_key(), vec![1_u8]), + (StoreKeys::libp2p_keypair(), vec![2_u8]), + ]) + .await?; + store + .scope(StoreKeys::sortition()) + .write_sync(7_u64) + .await?; + let repositories = Repositories::from(&store); + let eventstore = system.eventstore_reader()?.seq(); + + assert!(has_schema_governed_kv_state(&repositories).await?); + let error = preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) + .await + .unwrap_err(); + + assert!(error.to_string().contains("no schema marker")); + assert_eq!(repositories.schema_version().read().await?, None); + Ok(()) +} + +#[actix::test] +async fn schema_preflight_rejects_partial_bootstrap_identity() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let store = system.store()?; + store + .scope(StoreKeys::eth_private_key()) + .write_sync(vec![1_u8]) + .await?; + let repositories = Repositories::from(&store); + let eventstore = system.eventstore_reader()?.seq(); + + let error = preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) + .await + .unwrap_err(); + + assert!(error.to_string().contains("no schema marker")); + assert_eq!(repositories.schema_version().read().await?, None); + Ok(()) +} + +#[actix::test] +async fn schema_preflight_rejects_unversioned_event_log() -> anyhow::Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let store = system.store()?; + let repositories = Repositories::from(&store); + let bus = system.handle()?.enable("unversioned-event-log"); + bus.publish_without_context(e3_events::TestEvent::new("legacy", 1))?; + bus.flush_event_pipeline().await?; + let eventstore = system.eventstore_reader()?.seq(); + + let error = preflight_schema_version(&repositories, &system.aggregate_config(), &eventstore) + .await + .unwrap_err(); + + assert!(error.to_string().contains("no schema marker")); + Ok(()) +} diff --git a/crates/sync/src/domain/sync_planner.rs b/crates/sync/src/sync/workflow.rs similarity index 100% rename from crates/sync/src/domain/sync_planner.rs rename to crates/sync/src/sync/workflow.rs diff --git a/crates/zk-prover/src/actors/accusation_manager.rs b/crates/zk-prover/src/accusation_manager.rs similarity index 100% rename from crates/zk-prover/src/actors/accusation_manager.rs rename to crates/zk-prover/src/accusation_manager.rs diff --git a/crates/zk-prover/src/actors/accusation_manager_ext.rs b/crates/zk-prover/src/accusation_manager_ext.rs similarity index 100% rename from crates/zk-prover/src/actors/accusation_manager_ext.rs rename to crates/zk-prover/src/accusation_manager_ext.rs diff --git a/crates/zk-prover/src/actor_system.rs b/crates/zk-prover/src/actor_system.rs new file mode 100644 index 0000000000..03761cf686 --- /dev/null +++ b/crates/zk-prover/src/actor_system.rs @@ -0,0 +1,85 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Startup composition for the global ZK actor system. + +use actix::{Actor, Addr}; +use alloy::primitives::Address; +use alloy::signers::local::PrivateKeySigner; +use e3_events::{BusHandle, Committee, E3id}; +use e3_request::E3Meta; +use std::collections::HashMap; + +use crate::actors::{ + NodeProofAggregator, ProofRequestActor, ProofVerificationActor, ShareVerificationActor, ZkActor, +}; +use crate::ZkBackend; + +/// Durable inputs needed by global proof-verification actors before EventStore replay begins. +/// +/// Both maps are projections of canonical protocol events. They are startup seeds, not separate +/// authorities: live or replayed lifecycle events continue to update the actor caches. +#[derive(Clone, Debug, Default)] +pub struct ZkActorRecovery { + finalized_committees: HashMap, + e3_metadata: HashMap, +} + +impl ZkActorRecovery { + pub fn new( + finalized_committees: HashMap, + e3_metadata: HashMap, + ) -> Self { + Self { + finalized_committees, + e3_metadata, + } + } +} + +/// Setup all ZK-related actors. +/// +/// Requires a `ZkBackend` for proof generation/verification and a `PrivateKeySigner` for signing +/// proofs. `dkg_fold_attestation_verifiers_by_chain` maps each enabled chain to its EIP-712 +/// verifying contract. `recovery` seeds global verifier context before EventStore replay. +pub fn setup_zk_actors( + bus: &BusHandle, + backend: &ZkBackend, + signer: PrivateKeySigner, + dkg_fold_attestation_verifiers_by_chain: HashMap>, + recovery: ZkActorRecovery, +) -> ZkActors { + let ZkActorRecovery { + finalized_committees, + e3_metadata, + } = recovery; + let zk_actor = ZkActor::new(backend).start(); + let verifier = zk_actor.clone().recipient(); + + let proof_request = ProofRequestActor::setup(bus, signer.clone()); + let proof_verification = + ProofVerificationActor::setup(bus, verifier, finalized_committees.clone(), e3_metadata); + let share_verification = ShareVerificationActor::setup(bus, finalized_committees); + let node_proof_aggregator = + NodeProofAggregator::setup(bus, signer, dkg_fold_attestation_verifiers_by_chain); + + ZkActors { + zk_actor, + proof_request, + proof_verification, + share_verification, + node_proof_aggregator, + } +} + +/// Container for all ZK actor addresses. +pub struct ZkActors { + pub zk_actor: Addr, + pub proof_request: Addr, + pub proof_verification: Addr, + pub share_verification: Addr, + pub node_proof_aggregator: Addr, +} diff --git a/crates/zk-prover/src/actors.rs b/crates/zk-prover/src/actors.rs new file mode 100644 index 0000000000..dca7715d1d --- /dev/null +++ b/crates/zk-prover/src/actors.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Compatibility view of actors stored with their ZK capabilities. +//! +//! Startup composition lives in [`crate::actor_system`]. + +#[path = "accusation_manager.rs"] +pub mod accusation_manager; +#[path = "accusation_manager_ext.rs"] +pub mod accusation_manager_ext; +#[path = "commitment_consistency_checker.rs"] +pub mod commitment_consistency_checker; +#[path = "commitment_consistency_checker_ext.rs"] +pub mod commitment_consistency_checker_ext; +#[path = "node_proof_aggregation/actor.rs"] +pub mod node_proof_aggregator; +#[path = "proof_request/actor.rs"] +pub mod proof_request; +#[path = "proof_verification/actor.rs"] +pub mod proof_verification; +#[path = "share_verification/actor.rs"] +pub mod share_verification; +#[path = "zk_actor.rs"] +pub mod zk_actor; + +// Re-export accusation types from their canonical home in e3-slashing. +pub use e3_slashing::CommitmentConsistencyCheckerExtension; +pub use node_proof_aggregator::NodeProofAggregator; +pub use proof_request::ProofRequestActor; +pub use proof_verification::{ + ProofVerificationActor, ZkVerificationRequest, ZkVerificationResponse, +}; +pub use share_verification::ShareVerificationActor; +pub use zk_actor::ZkActor; diff --git a/crates/zk-prover/src/actors/mod.rs b/crates/zk-prover/src/actors/mod.rs deleted file mode 100644 index 19f4901436..0000000000 --- a/crates/zk-prover/src/actors/mod.rs +++ /dev/null @@ -1,140 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Actor-based components for ZK proof generation and verification. -//! -//! ## Architecture -//! -//! This module follows a clean separation between core business logic and IO operations: -//! -//! ### Core Actors (Business Logic - No IO) -//! - [`ProofRequestActor`]: Converts `EncryptionKeyPending` → `ComputeRequest` and handles responses -//! - [`ProofVerificationActor`]: Verifies `EncryptionKeyReceived` and converts to `EncryptionKeyCreated` -//! - [`ShareVerificationActor`]: Handles ECDSA + ZK verification for C2/C3/C4 share proofs -//! -//! ### IO Actors (File System Operations) -//! - [`ZkActor`]: Performs actual proof generation/verification using disk-based circuits and bb binary -//! -//! ## Usage -//! -//! ```rust,ignore -//! use e3_zk_prover::{ZkActorRecovery, ZkBackend, setup_zk_actors}; -//! use e3_events::BusHandle; -//! use alloy::signers::local::PrivateKeySigner; -//! use std::collections::HashMap; -//! -//! let bus = BusHandle::default(); -//! let backend = ZkBackend::with_default_dir().await?; -//! let signer = PrivateKeySigner::random(); -//! -//! // Setup all actors with proper separation of concerns -//! setup_zk_actors( -//! &bus, -//! &backend, -//! signer, -//! HashMap::new(), -//! ZkActorRecovery::default(), -//! ); -//! ``` - -pub mod accusation_manager; -pub mod accusation_manager_ext; -pub mod commitment_consistency_checker; -pub mod commitment_consistency_checker_ext; -pub mod commitment_links; -pub mod node_proof_aggregator; -pub mod proof_request; -pub mod proof_verification; -pub mod share_verification; -pub mod zk_actor; - -// Re-export accusation types from their canonical home in e3-slashing. -pub use e3_slashing::CommitmentConsistencyCheckerExtension; -pub use node_proof_aggregator::NodeProofAggregator; -pub use proof_request::ProofRequestActor; -pub use proof_verification::{ - ProofVerificationActor, ZkVerificationRequest, ZkVerificationResponse, -}; -pub use share_verification::ShareVerificationActor; -pub use zk_actor::ZkActor; - -use actix::{Actor, Addr}; -use alloy::primitives::Address; -use alloy::signers::local::PrivateKeySigner; -use e3_events::{BusHandle, Committee, E3id}; -use e3_request::E3Meta; -use std::collections::HashMap; - -use crate::ZkBackend; - -/// Durable inputs needed by global proof-verification actors before EventStore replay begins. -/// -/// Both maps are projections of canonical protocol events. They are startup seeds, not separate -/// authorities: live or replayed lifecycle events continue to update the actor caches. -#[derive(Clone, Debug, Default)] -pub struct ZkActorRecovery { - finalized_committees: HashMap, - e3_metadata: HashMap, -} - -impl ZkActorRecovery { - pub fn new( - finalized_committees: HashMap, - e3_metadata: HashMap, - ) -> Self { - Self { - finalized_committees, - e3_metadata, - } - } -} - -/// Setup all ZK-related actors with proper separation of concerns. -/// -/// Requires a `ZkBackend` for proof generation/verification and a -/// `PrivateKeySigner` for signing proofs (fault attribution). -/// `dkg_fold_attestation_verifiers_by_chain` maps each enabled chain's id to -/// `CiphernodeRegistry.dkgFoldAttestationVerifier()` (EIP-712 `verifyingContract` -/// for fold attestations). Fetched at node startup when proof aggregation is enabled. -/// `recovery` seeds global verifier context before EventStore replay. -pub fn setup_zk_actors( - bus: &BusHandle, - backend: &ZkBackend, - signer: PrivateKeySigner, - dkg_fold_attestation_verifiers_by_chain: HashMap>, - recovery: ZkActorRecovery, -) -> ZkActors { - let ZkActorRecovery { - finalized_committees, - e3_metadata, - } = recovery; - let zk_actor = ZkActor::new(backend).start(); - let verifier = zk_actor.clone().recipient(); - - let proof_request = ProofRequestActor::setup(bus, signer.clone()); - let proof_verification = - ProofVerificationActor::setup(bus, verifier, finalized_committees.clone(), e3_metadata); - let share_verification = ShareVerificationActor::setup(bus, finalized_committees); - let node_proof_aggregator = - NodeProofAggregator::setup(bus, signer, dkg_fold_attestation_verifiers_by_chain); - - ZkActors { - zk_actor, - proof_request, - proof_verification, - share_verification, - node_proof_aggregator, - } -} - -/// Container for all ZK-related actor addresses. -pub struct ZkActors { - pub zk_actor: Addr, - pub proof_request: Addr, - pub proof_verification: Addr, - pub share_verification: Addr, - pub node_proof_aggregator: Addr, -} diff --git a/crates/zk-prover/src/actors/node_proof_aggregator.rs b/crates/zk-prover/src/actors/node_proof_aggregator.rs deleted file mode 100644 index e0403f3064..0000000000 --- a/crates/zk-prover/src/actors/node_proof_aggregator.rs +++ /dev/null @@ -1,717 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Node-level DKG proof aggregation: buffer all inner proofs (C0–C4), then run one -//! [`ZkRequest::NodeDkgFold`] when [`ThresholdSharePending`] says the full set is ready. - -use std::collections::{BTreeMap, HashMap}; - -use actix::{Actor, Addr, Context, Handler}; -use alloy::primitives::Address; -use alloy::signers::local::PrivateKeySigner; -use e3_events::{ - BusHandle, ComputeRequest, ComputeRequestError, ComputeResponse, ComputeResponseKind, - CorrelationId, DKGInnerProofReady, DKGRecursiveAggregationComplete, DkgFoldAttestationPayload, - E3Failed, E3Stage, E3id, EventContext, EventPublisher, EventSubscriber, EventType, - FailureReason, InterfoldEvent, InterfoldEventData, Proof, Sequenced, SignedDkgFoldAttestation, - ThresholdSharePending, TypedEvent, ZkRequest, ZkResponse, -}; -use e3_fhe_params::build_pair_for_preset; -use tracing::{error, info, warn}; - -use crate::domain::node_dkg_fold::{DkgProofCollectionState, NodeDkgFoldMeta}; -use crate::node_fold_public::extract_node_fold_agg_commits; - -/// Actor that collects DKG inner proofs and dispatches a single [`ZkRequest::NodeDkgFold`]. -pub struct NodeProofAggregator { - bus: BusHandle, - signer: PrivateKeySigner, - /// Per-chain `DkgFoldAttestationVerifier` address (EIP-712 `verifyingContract`). - /// Looked up by `e3_id.chain_id()` when signing fold attestations. - dkg_fold_attestation_verifiers_by_chain: HashMap>, - states: HashMap, - fold_correlation: HashMap, - pending_inner_proofs: HashMap>, -} - -impl NodeProofAggregator { - pub fn new( - bus: &BusHandle, - signer: PrivateKeySigner, - dkg_fold_attestation_verifiers_by_chain: HashMap>, - ) -> Self { - Self { - bus: bus.clone(), - signer, - dkg_fold_attestation_verifiers_by_chain, - states: HashMap::new(), - fold_correlation: HashMap::new(), - pending_inner_proofs: HashMap::new(), - } - } - - pub fn setup( - bus: &BusHandle, - signer: PrivateKeySigner, - dkg_fold_attestation_verifiers_by_chain: HashMap>, - ) -> Addr { - let addr = Self::new(bus, signer, dkg_fold_attestation_verifiers_by_chain).start(); - bus.subscribe(EventType::ThresholdSharePending, addr.clone().into()); - bus.subscribe(EventType::DKGInnerProofReady, addr.clone().into()); - bus.subscribe(EventType::ComputeResponse, addr.clone().into()); - bus.subscribe(EventType::ComputeRequestError, addr.clone().into()); - addr - } - - fn handle_threshold_share_pending(&mut self, msg: TypedEvent) { - let (msg, ec) = msg.into_components(); - let e3_id = msg.e3_id.clone(); - - if !msg.proof_aggregation_enabled { - self.pending_inner_proofs.remove(&e3_id); - info!( - "NodeProofAggregator: proof aggregation disabled for E3 {} — skipping", - e3_id - ); - if let Err(err) = self.bus.publish( - DKGRecursiveAggregationComplete { - e3_id: e3_id.clone(), - party_id: msg.full_share.party_id, - aggregated_proof: None, - fold_attestation: None, - }, - ec, - ) { - error!( - "NodeProofAggregator: failed to publish DKGRecursiveAggregationComplete (skipped) for E3 {}: {err}", - e3_id - ); - } - return; - } - - let sk_enc_count = msg.sk_share_encryption_requests.len(); - let e_sm_enc_count = msg.e_sm_share_encryption_requests.len(); - let total_expected = NodeDkgFoldMeta::total_expected_for(sk_enc_count, e_sm_enc_count); - - let committee = msg.proof_request.committee_size.values(); - let (committee_n, committee_h, n_moduli) = - match build_pair_for_preset(msg.proof_request.params_preset) { - Ok((threshold_params, _)) => { - (committee.n, committee.h, threshold_params.moduli().len()) - } - Err(e) => { - self.pending_inner_proofs.remove(&e3_id); - error!( - "NodeProofAggregator: build_pair_for_preset failed for E3 {}: {e}", - e3_id - ); - let _ = self.bus.publish( - E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DKGInvalidShares, - }, - ec.clone(), - ); - return; - } - }; - - let meta = NodeDkgFoldMeta { - party_id: msg.full_share.party_id, - total_expected, - sk_enc_count, - e_sm_enc_count, - sk_share_encryption_requests: msg.sk_share_encryption_requests.clone(), - e_sm_share_encryption_requests: msg.e_sm_share_encryption_requests.clone(), - committee_n, - committee_h, - n_moduli, - params_preset: msg.proof_request.params_preset, - committee_size: msg.proof_request.committee_size, - }; - - info!( - "NodeProofAggregator: E3 {} party {} — expecting {} inner proofs (C0..C4) for NodeDkgFold", - e3_id, meta.party_id, total_expected, - ); - - self.initialize_collection_state(e3_id, meta, ec); - } - - fn handle_inner_proof_ready(&mut self, msg: TypedEvent) { - let (msg, ec) = msg.into_components(); - let e3_id = msg.e3_id.clone(); - - let Some(state) = self.states.get_mut(&e3_id) else { - let pending = self.pending_inner_proofs.entry(e3_id.clone()).or_default(); - pending.insert(msg.seq, msg.proof); - warn!( - "NodeProofAggregator: received DKGInnerProofReady for E3 {} before ThresholdSharePending — prebuffered seq={} (have {})", - e3_id, - msg.seq, - pending.len() - ); - return; - }; - - if state.fold_correlation.is_some() { - warn!( - "NodeProofAggregator: seq={} arrived while NodeDkgFold in flight — dropped", - msg.seq - ); - return; - } - - state.buffer.insert(msg.seq, msg.proof); - state.last_ec = ec; - - info!( - "NodeProofAggregator: buffered seq={} for E3 {} (have {}/{})", - msg.seq, - e3_id, - state.buffer.len(), - state.meta.total_expected - ); - - self.try_dispatch_node_dkg_fold(&e3_id); - } - - fn initialize_collection_state( - &mut self, - e3_id: E3id, - meta: NodeDkgFoldMeta, - ec: EventContext, - ) { - let mut buffer = self.pending_inner_proofs.remove(&e3_id).unwrap_or_default(); - if !buffer.is_empty() { - info!( - "NodeProofAggregator: recovered {} prebuffered inner proofs for E3 {}", - buffer.len(), - e3_id - ); - } - - self.states.insert( - e3_id.clone(), - DkgProofCollectionState::new(meta, std::mem::take(&mut buffer), ec), - ); - - self.try_dispatch_node_dkg_fold(&e3_id); - } - - fn try_dispatch_node_dkg_fold(&mut self, e3_id: &E3id) { - let state = match self.states.get_mut(e3_id) { - Some(s) => s, - None => return, - }; - if !state.is_ready() { - return; - } - - let req = state.build_fold_request(); - let corr = CorrelationId::new(); - let ec = state.last_ec.clone(); - let party_id = state.meta.party_id; - - state.fold_correlation = Some(corr); - self.fold_correlation.insert(corr, e3_id.clone()); - - info!( - "NodeProofAggregator: dispatching NodeDkgFold for E3 {} party {}", - e3_id, party_id - ); - - if let Err(err) = self.bus.publish( - ComputeRequest::zk(ZkRequest::NodeDkgFold(req), corr, e3_id.clone()), - ec, - ) { - error!( - "NodeProofAggregator: failed to publish NodeDkgFold for E3 {}: {err}", - e3_id - ); - let _ = self.states.get_mut(e3_id).map(|s| { - s.fold_correlation = None; - }); - self.fold_correlation.remove(&corr); - } - } - - fn handle_node_dkg_response(&mut self, correlation_id: &CorrelationId, proof: Proof) { - let Some(e3_id) = self.fold_correlation.remove(correlation_id) else { - return; - }; - - let Some(state) = self.states.remove(&e3_id) else { - error!( - "NodeProofAggregator: NodeDkgFold response for unknown E3 {}", - e3_id - ); - return; - }; - - let party_id = state.meta.party_id; - let committee_n = state.meta.committee_n; - let committee_h = state.meta.committee_h; - let n_moduli = state.meta.n_moduli; - - let fold_attestation = match extract_node_fold_agg_commits( - &proof, - committee_n, - committee_h, - n_moduli, - ) { - Ok((extracted_party, commits)) => { - if extracted_party != party_id { - error!( - e3_id = %e3_id, - expected_party_id = party_id, - extracted_party_id = extracted_party, - "NodeFold public party_id does not match sortition party_id" - ); - None - } else if let Some(verifying_contract) = - self.dkg_fold_attestation_verifier_for(&e3_id) - { - let payload = DkgFoldAttestationPayload { - e3_id: e3_id.clone(), - verifying_contract, - party_id, - agg_commits: commits, - }; - match SignedDkgFoldAttestation::sign(payload, &self.signer) { - Ok(signed) => Some(signed), - Err(e) => { - error!( - e3_id = %e3_id, - party_id, - error = %e, - "failed to sign DkgFoldAttestation" - ); - None - } - } - } else { - error!( - e3_id = %e3_id, - party_id, - "NodeProofAggregator: cannot sign DkgFoldAttestation — CiphernodeRegistry.dkgFoldAttestationVerifier not configured" - ); - None - } - } - Err(e) => { - error!( - e3_id = %e3_id, - party_id, - error = %e, - "failed to extract sk_agg/esm_agg from NodeFold proof" - ); - None - } - }; - - if fold_attestation.is_none() { - error!( - e3_id = %e3_id, - party_id, - "NodeDkgFold succeeded but fold attestation missing — failing E3" - ); - if let Err(err) = self.bus.publish( - E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DKGInvalidShares, - }, - state.last_ec, - ) { - error!( - "NodeProofAggregator: failed to publish E3Failed for E3 {}: {err}", - e3_id - ); - } - return; - } - - info!( - "NodeProofAggregator: NodeDkgFold complete for E3 {} party {} — publishing DKGRecursiveAggregationComplete", - e3_id, party_id - ); - - if let Err(err) = self.bus.publish( - DKGRecursiveAggregationComplete { - e3_id: e3_id.clone(), - party_id, - aggregated_proof: Some(proof), - fold_attestation, - }, - state.last_ec, - ) { - error!( - "NodeProofAggregator: failed to publish DKGRecursiveAggregationComplete for E3 {}: {err}", - e3_id - ); - } - } -} - -impl Actor for NodeProofAggregator { - type Context = Context; -} - -impl Handler for NodeProofAggregator { - type Result = (); - - fn handle(&mut self, msg: InterfoldEvent, _ctx: &mut Self::Context) -> Self::Result { - let (data, ec) = msg.into_components(); - match data { - InterfoldEventData::ThresholdSharePending(data) => { - self.handle_threshold_share_pending(TypedEvent::new(data, ec)); - } - InterfoldEventData::DKGInnerProofReady(data) => { - self.handle_inner_proof_ready(TypedEvent::new(data, ec)); - } - InterfoldEventData::ComputeResponse(data) => { - self.handle_compute_response(TypedEvent::new(data, ec)); - } - InterfoldEventData::ComputeRequestError(data) => { - self.handle_compute_request_error(TypedEvent::new(data, ec)); - } - _ => {} - } - } -} - -impl Handler> for NodeProofAggregator { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_threshold_share_pending(msg); - } -} - -impl Handler> for NodeProofAggregator { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_inner_proof_ready(msg); - } -} - -impl Handler> for NodeProofAggregator { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_compute_response(msg); - } -} - -impl Handler> for NodeProofAggregator { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_compute_request_error(msg); - } -} - -impl NodeProofAggregator { - fn dkg_fold_attestation_verifier_for(&self, e3_id: &E3id) -> Option
{ - let chain_id = e3_id.chain_id(); - match self.dkg_fold_attestation_verifiers_by_chain.get(&chain_id) { - Some(Some(addr)) => Some(*addr), - Some(None) => None, - None => { - warn!( - chain_id, - "no dkgFoldAttestationVerifier configured for chain" - ); - None - } - } - } - - fn handle_compute_response(&mut self, msg: TypedEvent) { - let (msg, _ec) = msg.into_components(); - if let ComputeResponseKind::Zk(ZkResponse::NodeDkgFold(resp)) = msg.response { - self.handle_node_dkg_response(&msg.correlation_id, resp.proof); - } - } - - fn handle_compute_request_error(&mut self, msg: TypedEvent) { - let (msg, ec) = msg.into_components(); - if let Some(e3_id) = self.fold_correlation.remove(msg.correlation_id()) { - error!( - "NodeProofAggregator: NodeDkgFold failed for E3 {}: {:?} — aggregation aborted", - e3_id, - msg.get_err() - ); - let state = self.states.remove(&e3_id); - warn!( - "NodeProofAggregator: E3 {} NodeDkgFold failed — publishing E3Failed", - e3_id - ); - - if let Some(_state) = state { - if let Err(err) = self.bus.publish( - E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DKGInvalidShares, - }, - ec, - ) { - error!( - "NodeProofAggregator: failed to publish E3Failed for E3 {}: {err}", - e3_id - ); - } - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use anyhow::Result; - use e3_events::{ - CircuitName, ComputeRequestErrorKind, ComputeRequestKind, Event, HistoryCollector, - NodeDkgFoldRequest, TakeEvents, Unsequenced, ZkError, - }; - use e3_test_helpers::get_common_setup; - use e3_zk_helpers::CiphernodesCommitteeSize; - - fn test_ctx(data: impl Into) -> EventContext { - EventContext::::from(data.into()).sequence(0) - } - - fn test_signer() -> PrivateKeySigner { - "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" - .parse() - .expect("test signer") - } - - fn dummy_proof(seed: u8) -> Proof { - Proof::new( - CircuitName::PkAggregation, - e3_utils::ArcBytes::from_bytes(&[seed]), - e3_utils::ArcBytes::from_bytes(&[seed.wrapping_add(1)]), - ) - } - - async fn next_event( - history: &Addr>, - ) -> Result { - let mut result = history.send(TakeEvents::::new(1)).await?; - assert!(!result.timed_out, "timed out waiting for an event"); - Ok(result.events.pop().expect("expected one event")) - } - - #[actix::test] - async fn node_dkg_fold_compute_error_emits_e3_failed() -> Result<()> { - let (bus, _rng, _seed, _params, _crp, _errors, history) = get_common_setup(None)?; - let mut aggregator = NodeProofAggregator::new(&bus, test_signer(), HashMap::new()); - let e3_id = E3id::new("42", 1); - let correlation_id = CorrelationId::new(); - - aggregator.states.insert( - e3_id.clone(), - DkgProofCollectionState { - meta: NodeDkgFoldMeta { - party_id: 7, - total_expected: 0, - sk_enc_count: 0, - e_sm_enc_count: 0, - sk_share_encryption_requests: Vec::new(), - e_sm_share_encryption_requests: Vec::new(), - committee_n: 0, - committee_h: 0, - n_moduli: 0, - params_preset: e3_fhe_params::BfvPreset::InsecureThreshold512, - committee_size: CiphernodesCommitteeSize::Minimum, - }, - buffer: BTreeMap::new(), - fold_correlation: Some(correlation_id), - last_ec: test_ctx(DKGRecursiveAggregationComplete { - e3_id: e3_id.clone(), - party_id: 7, - aggregated_proof: None, - fold_attestation: None, - }), - }, - ); - aggregator - .fold_correlation - .insert(correlation_id, e3_id.clone()); - - let request = ComputeRequest::zk( - ZkRequest::NodeDkgFold(NodeDkgFoldRequest { - c0_proof: dummy_proof(1), - c1_proof: dummy_proof(2), - c2a_proof: dummy_proof(3), - c2b_proof: dummy_proof(4), - c3a_inner_proofs: Vec::new(), - c3b_inner_proofs: Vec::new(), - c4a_proof: dummy_proof(5), - c4b_proof: dummy_proof(6), - c3_slot_indices_a: Vec::new(), - c3_slot_indices_b: Vec::new(), - c3_total_slots: 0, - party_id: 7, - params_preset: e3_fhe_params::BfvPreset::InsecureThreshold512, - committee_size: CiphernodesCommitteeSize::Minimum, - }), - correlation_id, - e3_id.clone(), - ); - - aggregator.handle_compute_request_error(TypedEvent::new( - ComputeRequestError::new( - ComputeRequestErrorKind::Zk(ZkError::ProofGenerationFailed("boom".to_string())), - request, - ), - test_ctx(DKGRecursiveAggregationComplete { - e3_id: e3_id.clone(), - party_id: 7, - aggregated_proof: None, - fold_attestation: None, - }), - )); - - let event = next_event(&history).await?; - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == e3_id - && data.failed_at_stage == E3Stage::CommitteeFinalized - && data.reason == FailureReason::DKGInvalidShares - )); - assert!(!aggregator.states.contains_key(&e3_id)); - assert!(aggregator.fold_correlation.is_empty()); - - Ok(()) - } - - #[actix::test] - async fn early_inner_proof_is_prebuffered_until_collection_starts() -> Result<()> { - let (bus, _rng, _seed, _params, _crp, _errors, history) = get_common_setup(None)?; - let mut aggregator = NodeProofAggregator::new(&bus, test_signer(), HashMap::new()); - let e3_id = E3id::new("43", 1); - let early_proof = dummy_proof(10); - - aggregator.handle_inner_proof_ready(TypedEvent::new( - DKGInnerProofReady { - e3_id: e3_id.clone(), - party_id: 7, - proof: early_proof.clone(), - seq: 0, - }, - test_ctx(DKGInnerProofReady { - e3_id: e3_id.clone(), - party_id: 7, - proof: early_proof.clone(), - seq: 0, - }), - )); - - assert_eq!( - aggregator - .pending_inner_proofs - .get(&e3_id) - .map(BTreeMap::len), - Some(1) - ); - - aggregator.initialize_collection_state( - e3_id.clone(), - NodeDkgFoldMeta { - party_id: 7, - total_expected: 6, - sk_enc_count: 0, - e_sm_enc_count: 0, - sk_share_encryption_requests: Vec::new(), - e_sm_share_encryption_requests: Vec::new(), - committee_n: 0, - committee_h: 0, - n_moduli: 0, - params_preset: e3_fhe_params::BfvPreset::InsecureThreshold512, - committee_size: CiphernodesCommitteeSize::Minimum, - }, - test_ctx(DKGRecursiveAggregationComplete { - e3_id: e3_id.clone(), - party_id: 7, - aggregated_proof: None, - fold_attestation: None, - }), - ); - - assert!(!aggregator.pending_inner_proofs.contains_key(&e3_id)); - assert_eq!( - aggregator - .states - .get(&e3_id) - .map(|state| state.buffer.len()), - Some(1) - ); - - for seq in 1..6 { - let proof = dummy_proof((10 + seq) as u8); - aggregator.handle_inner_proof_ready(TypedEvent::new( - DKGInnerProofReady { - e3_id: e3_id.clone(), - party_id: 7, - proof: proof.clone(), - seq, - }, - test_ctx(DKGInnerProofReady { - e3_id: e3_id.clone(), - party_id: 7, - proof, - seq, - }), - )); - } - - let event = next_event(&history).await?; - match event.into_data() { - InterfoldEventData::ComputeRequest(request) => { - assert_eq!(request.e3_id, e3_id); - match request.request { - ComputeRequestKind::Zk(ZkRequest::NodeDkgFold(fold_request)) => { - assert_eq!(fold_request.c0_proof, early_proof); - } - other => panic!("expected NodeDkgFold request, got {other:?}"), - } - } - other => panic!("expected ComputeRequest event, got {other:?}"), - } - - assert!(aggregator - .states - .get(&e3_id) - .and_then(|state| state.fold_correlation) - .is_some()); - - Ok(()) - } -} diff --git a/crates/zk-prover/src/actors/proof_request.rs b/crates/zk-prover/src/actors/proof_request.rs deleted file mode 100644 index 2e0f573386..0000000000 --- a/crates/zk-prover/src/actors/proof_request.rs +++ /dev/null @@ -1,1444 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -use std::collections::{BTreeMap, HashMap}; -use std::sync::Arc; - -use actix::{Actor, Addr, Context, Handler}; -use alloy::primitives::{keccak256, Bytes}; -use alloy::signers::local::PrivateKeySigner; -use alloy::sol_types::SolValue; -use e3_events::{ - AggregationProofPending, AggregationProofSigned, BusHandle, ComputeRequest, - ComputeRequestError, ComputeRequestErrorKind, ComputeResponse, ComputeResponseKind, - CorrelationId, DKGInnerProofReady, DecryptionKeyShared, DecryptionShareProofSigned, - DecryptionShareProofsPending, DecryptionshareCreated, DkgProofSigned, E3Failed, E3Stage, E3id, - EncryptionKeyCreated, EncryptionKeyPending, EventContext, EventPublisher, EventSubscriber, - EventType, FailureReason, InterfoldEvent, InterfoldEventData, PkAggregationProofPending, - PkAggregationProofSigned, PkBfvProofRequest, PkGenerationProofSigned, Proof, ProofPayload, - ProofType, ProofVerificationPassed, Sequenced, ShareDecryptionProofPending, SignedProofPayload, - ThresholdShareCreated, ThresholdSharePending, TypedEvent, ZkRequest, ZkResponse, -}; -use e3_utils::NotifySync; -use tracing::{error, info, trace, warn}; - -use crate::domain::proof_request::{ - plan_decryption_dispatch, plan_threshold_dispatch, DecryptionProofKind, NodeAggregationMeta, - PendingAggregationProof, PendingDecryptionProofs, PendingPkAggregationProof, - PendingProofRequest, PendingShareDecryptionProof, PendingThresholdProofs, ThresholdProofKind, -}; - -/// Core actor that handles encryption key proof requests. -/// -/// Proofs are always wrapped in a [`SignedProofPayload`] before being published, -/// enabling fault attribution via the signed proof model. -/// A signer is required — if signing fails, the proof is not published. -pub struct ProofRequestActor { - bus: BusHandle, - signer: PrivateKeySigner, - pending: HashMap, - threshold_correlation: HashMap, - pending_threshold: HashMap, - /// C4 proof staging: correlation -> (e3_id, kind, seq) - decryption_correlation: HashMap, - /// Per-E3 metadata for DKGInnerProofReady emission. - node_agg_meta: HashMap, - /// C4 pending proofs per E3 - pending_decryption: HashMap, - /// C6 proof staging: correlation -> e3_id - share_decryption_correlation: HashMap, - /// C6 pending proofs per E3 - pending_share_decryption: HashMap, - /// C5 proof staging: correlation -> e3_id - pk_aggregation_correlation: HashMap, - /// C5 pending proofs per E3 - pending_pk_aggregation: HashMap, - /// C7 proof staging: correlation -> e3_id - aggregation_correlation: HashMap, - /// C7 pending proofs per E3 - pending_aggregation: HashMap, -} - -impl ProofRequestActor { - pub fn new(bus: &BusHandle, signer: PrivateKeySigner) -> Self { - Self { - bus: bus.clone(), - signer, - pending: HashMap::new(), - pending_threshold: HashMap::new(), - threshold_correlation: HashMap::new(), - decryption_correlation: HashMap::new(), - pending_decryption: HashMap::new(), - node_agg_meta: HashMap::new(), - share_decryption_correlation: HashMap::new(), - pending_share_decryption: HashMap::new(), - pk_aggregation_correlation: HashMap::new(), - pending_pk_aggregation: HashMap::new(), - aggregation_correlation: HashMap::new(), - pending_aggregation: HashMap::new(), - } - } - - pub fn setup(bus: &BusHandle, signer: PrivateKeySigner) -> Addr { - let addr = Self::new(bus, signer).start(); - bus.subscribe(EventType::EncryptionKeyPending, addr.clone().into()); - bus.subscribe(EventType::ComputeResponse, addr.clone().into()); - bus.subscribe(EventType::ComputeRequestError, addr.clone().into()); - bus.subscribe(EventType::ThresholdSharePending, addr.clone().into()); - bus.subscribe(EventType::DecryptionShareProofsPending, addr.clone().into()); - bus.subscribe(EventType::ShareDecryptionProofPending, addr.clone().into()); - bus.subscribe(EventType::PkAggregationProofPending, addr.clone().into()); - bus.subscribe(EventType::AggregationProofPending, addr.clone().into()); - addr - } - - fn handle_encryption_key_pending(&mut self, msg: TypedEvent) { - let (msg, ec) = msg.into_components(); - let correlation_id = CorrelationId::new(); - self.pending.insert( - correlation_id, - PendingProofRequest { - e3_id: msg.e3_id.clone(), - key: msg.key.clone(), - }, - ); - - let request = ComputeRequest::zk( - ZkRequest::PkBfv(PkBfvProofRequest::new( - msg.key.pk_bfv.clone(), - msg.params_preset, - msg.committee_size, - )), - correlation_id, - msg.e3_id, - ); - - info!("Requesting C0 proof generation"); - if let Err(err) = self.bus.publish(request, ec) { - error!("Failed to publish ZK proof request: {err}"); - self.pending.remove(&correlation_id); - } - } - - fn handle_threshold_share_pending(&mut self, msg: TypedEvent) { - let (msg, ec) = msg.into_components(); - let e3_id = msg.e3_id.clone(); - - let sk_enc_count = msg.sk_share_encryption_requests.len(); - let e_sm_enc_count = msg.e_sm_share_encryption_requests.len(); - - let total_expected = NodeAggregationMeta::total_expected_for(sk_enc_count, e_sm_enc_count); - let pending_c0 = self - .node_agg_meta - .get(&e3_id) - .and_then(|m| m.pending_c0.clone()); - self.node_agg_meta.insert( - e3_id.clone(), - NodeAggregationMeta { - party_id: msg.full_share.party_id, - total_expected, - pending_c0: None, - proof_aggregation_enabled: msg.proof_aggregation_enabled, - }, - ); - // If C0 proof arrived before meta, emit DKGInnerProofReady now - if let Some(c0_proof) = pending_c0 { - if msg.proof_aggregation_enabled { - if let Err(err) = self.bus.publish( - DKGInnerProofReady { - e3_id: e3_id.clone(), - party_id: msg.full_share.party_id, - proof: c0_proof, - seq: 0, - }, - ec.clone(), - ) { - error!("Failed to publish DKGInnerProofReady for C0: {err}"); - } - } - } - - self.pending_threshold.insert( - e3_id.clone(), - PendingThresholdProofs::new( - e3_id.clone(), - msg.full_share.clone(), - ec.clone(), - sk_enc_count, - e_sm_enc_count, - msg.recipient_party_ids, - ), - ); - - // C1/C2/C3: dispatch threshold proof requests in canonical seq order. - // Sequencing/kind assignment lives in the pure domain planner; the actor - // only allocates correlation ids, publishes, and rolls back on failure. - for item in plan_threshold_dispatch( - msg.proof_request, - msg.sk_share_computation_request, - msg.e_sm_share_computation_request, - msg.sk_share_encryption_requests, - msg.e_sm_share_encryption_requests, - ) { - let corr = CorrelationId::new(); - self.threshold_correlation - .insert(corr, (e3_id.clone(), item.kind, item.seq)); - if let Err(err) = self.bus.publish( - ComputeRequest::zk(item.request, corr, e3_id.clone()), - ec.clone(), - ) { - error!("Failed to publish threshold proof request: {err}"); - self.threshold_correlation - .retain(|_, (eid, _, _)| *eid != e3_id); - self.pending_threshold.remove(&e3_id); - return; - } - } - } - - fn handle_compute_response(&mut self, msg: TypedEvent) { - let (msg, ec) = msg.into_components(); - match &msg.response { - ComputeResponseKind::Zk(ZkResponse::PkBfv(resp)) => { - self.handle_pk_bfv_response(&msg.correlation_id, resp.proof.clone(), &ec); - } - ComputeResponseKind::Zk(ZkResponse::PkGeneration(resp)) => { - self.handle_threshold_proof_response(&msg.correlation_id, resp.proof.clone(), &ec); - } - ComputeResponseKind::Zk(ZkResponse::ShareComputation(resp)) => { - self.handle_threshold_proof_response(&msg.correlation_id, resp.proof.clone(), &ec); - } - ComputeResponseKind::Zk(ZkResponse::ShareEncryption(resp)) => { - self.handle_threshold_proof_response(&msg.correlation_id, resp.proof.clone(), &ec); - } - ComputeResponseKind::Zk(ZkResponse::DkgShareDecryption(resp)) => { - // Try C4 decryption proof first, then fall back to C1/C2/C3 threshold - if self - .decryption_correlation - .contains_key(&msg.correlation_id) - { - self.handle_decryption_proof_response( - &msg.correlation_id, - resp.proof.clone(), - &ec, - ); - } else { - self.handle_threshold_proof_response( - &msg.correlation_id, - resp.proof.clone(), - &ec, - ); - } - } - ComputeResponseKind::Zk(ZkResponse::ThresholdShareDecryption(resp)) => { - self.handle_share_decryption_proof_response( - &msg.correlation_id, - resp.proofs.clone(), - ); - } - ComputeResponseKind::Zk(ZkResponse::PkAggregation(resp)) => { - self.handle_pk_aggregation_proof_response(&msg.correlation_id, resp.proof.clone()); - } - ComputeResponseKind::Zk(ZkResponse::DecryptedSharesAggregation(resp)) => { - self.handle_aggregation_proof_response(&msg.correlation_id, resp.proofs.clone()); - } - _ => {} - } - } - - /// Handle DecryptionShareProofsPending: dispatch C4 proof generation. - fn handle_decryption_share_proofs_pending( - &mut self, - msg: TypedEvent, - ) { - let (msg, ec) = msg.into_components(); - let e3_id = msg.e3_id.clone(); - let esm_count = msg.esm_requests.len(); - - if self.pending_decryption.contains_key(&e3_id) { - warn!( - "Duplicate DecryptionShareProofsPending for E3 {} — ignoring", - e3_id - ); - return; - } - - self.pending_decryption.insert( - e3_id.clone(), - PendingDecryptionProofs { - party_id: msg.party_id, - node: msg.node, - ec: ec.clone(), - sk_proof: None, - esm_proofs: HashMap::new(), - expected_esm_count: esm_count, - }, - ); - - // C4a/C4b: dispatch share-decryption proof requests in canonical seq - // order. The pure domain planner owns seq assignment; the actor only - // allocates correlation ids, publishes, and rolls back on failure. - let c4_base_seq = self - .node_agg_meta - .get(&e3_id) - .map(NodeAggregationMeta::c4_base_seq) - .unwrap_or(0); - for item in plan_decryption_dispatch(msg.sk_request, msg.esm_requests, c4_base_seq) { - let corr = CorrelationId::new(); - self.decryption_correlation - .insert(corr, (e3_id.clone(), item.kind, item.seq)); - if let Err(err) = self.bus.publish( - ComputeRequest::zk(item.request, corr, e3_id.clone()), - ec.clone(), - ) { - error!("Failed to publish C4 proof request: {err}"); - self.decryption_correlation - .retain(|_, (eid, _, _)| *eid != e3_id); - self.pending_decryption.remove(&e3_id); - return; - } - } - } - - /// Handle a C4 proof response — store and check completeness. - fn handle_decryption_proof_response( - &mut self, - correlation_id: &CorrelationId, - proof: Proof, - ec: &EventContext, - ) { - let Some((e3_id, kind, seq)) = self.decryption_correlation.remove(correlation_id) else { - return; - }; - - let Some(pending) = self.pending_decryption.get_mut(&e3_id) else { - error!( - "No pending decryption proofs for E3 {} — orphan correlation", - e3_id - ); - return; - }; - - let proof_for_agg = proof.clone(); - match kind { - DecryptionProofKind::SecretKey => { - info!("Received C4a SK decryption proof for E3 {}", e3_id); - pending.sk_proof = Some(proof); - } - DecryptionProofKind::SmudgingNoise { esi_idx } => { - info!( - "Received C4b ESM decryption proof [{}] for E3 {}", - esi_idx, e3_id - ); - pending.esm_proofs.insert(esi_idx, proof); - } - } - - if let Some(meta) = self.node_agg_meta.get(&e3_id) { - if meta.proof_aggregation_enabled { - if let Err(err) = self.bus.publish( - DKGInnerProofReady { - e3_id: e3_id.clone(), - party_id: meta.party_id, - proof: proof_for_agg, - seq, - }, - ec.clone(), - ) { - error!( - "Failed to publish DKGInnerProofReady for C4 seq={}: {err}", - seq - ); - } - } - } - - if pending.is_complete() { - info!( - "All C4 proofs complete for E3 {} — signing and publishing DecryptionKeyShared", - e3_id - ); - let pending = self.pending_decryption.remove(&e3_id).unwrap(); - self.sign_and_publish_decryption_key_shared(&e3_id, pending); - } - } - - /// Sign all C4 proofs and publish DecryptionKeyShared (Exchange #3). - fn sign_and_publish_decryption_key_shared( - &mut self, - e3_id: &E3id, - pending: PendingDecryptionProofs, - ) { - // Sign C4a (SK decryption proof) - let Some(signed_sk) = self.sign_proof( - e3_id, - ProofType::C4aSkShareDecryption, - pending.sk_proof.expect("checked in is_complete"), - ) else { - error!("Failed to sign C4a SK proof — DecryptionKeyShared will not be published"); - self.fail_dkg_round(e3_id.clone(), &pending.ec, "C4a signing error"); - return; - }; - - // Sign C4b (ESM decryption proofs) in esi_idx order - let mut signed_esms = Vec::with_capacity(pending.expected_esm_count); - for idx in 0..pending.expected_esm_count { - let proof = pending - .esm_proofs - .get(&idx) - .expect("checked in is_complete") - .clone(); - let Some(signed) = self.sign_proof(e3_id, ProofType::C4bESmShareDecryption, proof) - else { - error!( - "Failed to sign C4b ESM proof [{}] — DecryptionKeyShared will not be published", - idx - ); - self.fail_dkg_round(e3_id.clone(), &pending.ec, "C4b signing error"); - return; - }; - signed_esms.push(signed); - } - - info!( - "All C4 proofs signed for E3 {} party {} (signer: {})", - e3_id, - pending.party_id, - self.signer.address() - ); - - if let Err(err) = self.bus.publish( - DecryptionKeyShared { - e3_id: e3_id.clone(), - party_id: pending.party_id, - node: pending.node, - signed_sk_decryption_proof: signed_sk, - signed_e_sm_decryption_proofs: signed_esms, - external: false, - }, - pending.ec, - ) { - error!("Failed to publish DecryptionKeyShared: {err}"); - } - } - - /// Handle ShareDecryptionProofPending: dispatch C6 proof generation. - fn handle_share_decryption_proof_pending( - &mut self, - msg: TypedEvent, - ) { - let (msg, ec) = msg.into_components(); - let e3_id = msg.e3_id.clone(); - - if self.pending_share_decryption.contains_key(&e3_id) { - warn!( - "Duplicate ShareDecryptionProofPending for E3 {} — ignoring", - e3_id - ); - return; - } - - self.pending_share_decryption.insert( - e3_id.clone(), - PendingShareDecryptionProof { - party_id: msg.party_id, - node: msg.node, - decryption_share: msg.decryption_share, - ec: ec.clone(), - }, - ); - - let correlation_id = CorrelationId::new(); - self.share_decryption_correlation - .insert(correlation_id, e3_id.clone()); - - info!( - "Requesting C6 ThresholdShareDecryption proof for E3 {}", - e3_id - ); - if let Err(err) = self.bus.publish( - ComputeRequest::zk( - ZkRequest::ThresholdShareDecryption(msg.proof_request), - correlation_id, - e3_id.clone(), - ), - ec, - ) { - error!("Failed to publish C6 proof request: {err}"); - self.share_decryption_correlation.remove(&correlation_id); - self.pending_share_decryption.remove(&e3_id); - } - } - - /// Handle C6 proof response — sign proofs, publish DecryptionshareCreated. - fn handle_share_decryption_proof_response( - &mut self, - correlation_id: &CorrelationId, - proofs: Vec, - ) { - let Some(e3_id) = self.share_decryption_correlation.remove(correlation_id) else { - return; - }; - - let Some(pending) = self.pending_share_decryption.remove(&e3_id) else { - error!( - "No pending share decryption proof for E3 {} — orphan correlation", - e3_id - ); - return; - }; - - // Sign raw C6 proofs (for ShareVerification) - let mut signed_proofs = Vec::with_capacity(proofs.len()); - for proof in proofs { - let Some(signed) = - self.sign_proof(&e3_id, ProofType::C6ThresholdShareDecryption, proof) - else { - error!("Failed to sign C6 proof — DecryptionshareCreated will not be published"); - self.fail_decryption_round(e3_id.clone(), &pending.ec, "C6 signing error"); - return; - }; - signed_proofs.push(signed); - } - - info!( - "All C6 proofs signed for E3 {} party {} (signer: {})", - e3_id, - pending.party_id, - self.signer.address() - ); - - let ec = pending.ec; - - match self.bus.publish( - DecryptionshareCreated { - party_id: pending.party_id, - node: pending.node, - e3_id: e3_id.clone(), - decryption_share: pending.decryption_share, - signed_decryption_proofs: signed_proofs, - }, - ec.clone(), - ) { - Ok(_) => { - if let Err(err) = self.bus.publish( - DecryptionShareProofSigned { - e3_id: e3_id.clone(), - }, - ec, - ) { - error!("Failed to publish DecryptionShareProofSigned: {err}"); - } - } - Err(err) => { - error!("Failed to publish DecryptionshareCreated: {err}"); - } - } - } - - /// Handle PkAggregationProofPending: dispatch C5 proof generation. - fn handle_pk_aggregation_proof_pending(&mut self, msg: TypedEvent) { - let (msg, ec) = msg.into_components(); - let e3_id = msg.e3_id.clone(); - - if self.pending_pk_aggregation.contains_key(&e3_id) { - warn!( - "Duplicate PkAggregationProofPending for E3 {} — ignoring", - e3_id - ); - return; - } - - self.pending_pk_aggregation.insert( - e3_id.clone(), - PendingPkAggregationProof { - ec: ec.clone(), - request: msg.proof_request.clone(), - }, - ); - - let correlation_id = CorrelationId::new(); - self.pk_aggregation_correlation - .insert(correlation_id, e3_id.clone()); - - info!("Requesting C5 PkAggregation proof for E3 {}", e3_id); - if let Err(err) = self.bus.publish( - ComputeRequest::zk( - ZkRequest::PkAggregation(msg.proof_request), - correlation_id, - e3_id.clone(), - ), - ec, - ) { - error!("Failed to publish C5 proof request: {err}"); - self.pk_aggregation_correlation.remove(&correlation_id); - self.pending_pk_aggregation.remove(&e3_id); - } - } - - /// Handle C5 proof response — sign proof and publish PkAggregationProofSigned. - fn handle_pk_aggregation_proof_response( - &mut self, - correlation_id: &CorrelationId, - proof: Proof, - ) { - let Some(e3_id) = self.pk_aggregation_correlation.remove(correlation_id) else { - return; - }; - - let Some(pending) = self.pending_pk_aggregation.remove(&e3_id) else { - error!( - "No pending pk aggregation proof for E3 {} — orphan correlation", - e3_id - ); - return; - }; - - let Some(signed) = self.sign_proof(&e3_id, ProofType::C5PkAggregation, proof) else { - error!("Failed to sign C5 proof — PkAggregationProofSigned will not be published"); - self.fail_dkg_round(e3_id, &pending.ec, "C5 signing error"); - return; - }; - - info!( - "C5 proof signed for E3 {} (signer: {})", - e3_id, - self.signer.address() - ); - - if let Err(err) = self.bus.publish( - PkAggregationProofSigned { - e3_id: e3_id.clone(), - signed_proof: signed, - }, - pending.ec, - ) { - error!("Failed to publish PkAggregationProofSigned: {err}"); - } - } - - /// Handle AggregationProofPending: dispatch C7 proof generation. - fn handle_aggregation_proof_pending(&mut self, msg: TypedEvent) { - let (msg, ec) = msg.into_components(); - let e3_id = msg.e3_id.clone(); - - if self.pending_aggregation.contains_key(&e3_id) { - warn!( - "Duplicate AggregationProofPending for E3 {} — ignoring", - e3_id - ); - return; - } - - self.pending_aggregation - .insert(e3_id.clone(), PendingAggregationProof { ec: ec.clone() }); - - let correlation_id = CorrelationId::new(); - self.aggregation_correlation - .insert(correlation_id, e3_id.clone()); - - info!( - "Requesting C7 DecryptedSharesAggregation proof for E3 {}", - e3_id - ); - if let Err(err) = self.bus.publish( - ComputeRequest::zk( - ZkRequest::DecryptedSharesAggregation(msg.proof_request), - correlation_id, - e3_id.clone(), - ), - ec, - ) { - error!("Failed to publish C7 proof request: {err}"); - self.aggregation_correlation.remove(&correlation_id); - self.pending_aggregation.remove(&e3_id); - } - } - - /// Handle C7 proof response — sign proofs and publish AggregationProofSigned. - fn handle_aggregation_proof_response( - &mut self, - correlation_id: &CorrelationId, - proofs: Vec, - ) { - let Some(e3_id) = self.aggregation_correlation.remove(correlation_id) else { - return; - }; - - let Some(pending) = self.pending_aggregation.remove(&e3_id) else { - error!( - "No pending aggregation proof for E3 {} — orphan correlation", - e3_id - ); - return; - }; - - // Sign each C7 proof - let mut signed_proofs = Vec::with_capacity(proofs.len()); - for proof in proofs { - let Some(signed) = - self.sign_proof(&e3_id, ProofType::C7DecryptedSharesAggregation, proof) - else { - error!("Failed to sign C7 proof — AggregationProofSigned will not be published"); - self.fail_decryption_round(e3_id.clone(), &pending.ec, "C7 signing error"); - return; - }; - signed_proofs.push(signed); - } - - info!( - "All C7 proofs signed for E3 {} (signer: {})", - e3_id, - self.signer.address() - ); - - if let Err(err) = self.bus.publish( - AggregationProofSigned { - e3_id: e3_id.clone(), - signed_proofs, - }, - pending.ec, - ) { - error!("Failed to publish AggregationProofSigned: {err}"); - } - } - - fn handle_threshold_proof_response( - &mut self, - correlation_id: &CorrelationId, - proof: Proof, - ec: &EventContext, - ) { - let Some((e3_id, kind, seq)) = self.threshold_correlation.remove(correlation_id) else { - return; - }; - - let Some(pending) = self.pending_threshold.get_mut(&e3_id) else { - error!( - "No pending threshold proofs for E3 {} — orphan correlation", - e3_id - ); - return; - }; - - let proof_for_agg = proof.clone(); - pending.store_proof(&kind, proof); - info!( - "Received {:?} proof for E3 {} ({}/{})", - kind, - e3_id, - pending.total_received(), - pending.total_expected() - ); - - if let Some(meta) = self.node_agg_meta.get(&e3_id) { - if meta.proof_aggregation_enabled { - if let Err(err) = self.bus.publish( - DKGInnerProofReady { - e3_id: e3_id.clone(), - party_id: meta.party_id, - proof: proof_for_agg, - seq, - }, - ec.clone(), - ) { - error!( - "Failed to publish DKGInnerProofReady for {:?} seq={}: {err}", - kind, seq - ); - } - } - } - - if pending.is_complete() { - info!( - "All {} threshold proofs complete for E3 {}", - pending.total_expected(), - e3_id - ); - let pending = self.pending_threshold.remove(&e3_id).unwrap(); - self.publish_threshold_share_with_proofs(pending); - } - } - - fn sign_proof( - &self, - e3_id: &E3id, - proof_type: ProofType, - proof: Proof, - ) -> Option { - let payload = ProofPayload { - e3_id: e3_id.clone(), - proof_type, - proof, - }; - match SignedProofPayload::sign(payload, &self.signer) { - Ok(signed) => Some(signed), - Err(err) => { - error!("Failed to sign {:?} proof: {err}", proof_type); - None - } - } - } - - fn fail_dkg_round(&self, e3_id: E3id, ec: &EventContext, context: &str) { - if let Err(err) = self.bus.publish( - E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DKGInvalidShares, - }, - ec.clone(), - ) { - error!( - "Failed to publish E3Failed for {context} on E3 {}: {err}", - e3_id - ); - } - } - - fn fail_decryption_round(&self, e3_id: E3id, ec: &EventContext, context: &str) { - if let Err(err) = self.bus.publish( - E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CiphertextReady, - reason: FailureReason::DecryptionInvalidShares, - }, - ec.clone(), - ) { - error!( - "Failed to publish E3Failed for {context} on E3 {}: {err}", - e3_id - ); - } - } - - fn sign_and_group_proofs( - &self, - e3_id: &E3id, - proof_type: ProofType, - proofs: impl Iterator, - ) -> Option>> { - let mut map: BTreeMap> = BTreeMap::new(); - for (recipient, proof) in proofs { - let signed = self.sign_proof(e3_id, proof_type, proof)?; - map.entry(recipient).or_default().push(signed); - } - Some(map) - } - - fn publish_threshold_share_with_proofs(&mut self, pending: PendingThresholdProofs) { - let e3_id = &pending.e3_id; - let party_id = pending.full_share.party_id; - let ec = &pending.ec; - - // Sign C1 (PkGeneration) - let Some(signed_pk_gen) = self.sign_proof( - e3_id, - ProofType::C1PkGeneration, - pending.pk_generation_proof.expect("checked"), - ) else { - error!("Failed to sign C1 proof — shares will not be published"); - self.fail_dkg_round(e3_id.clone(), ec, "C1 signing error"); - return; - }; - - // Sign C2a (SkShareComputation) - let Some(signed_c2a) = self.sign_proof( - e3_id, - ProofType::C2aSkShareComputation, - pending.sk_share_computation_proof.expect("checked"), - ) else { - error!("Failed to sign C2a proof — shares will not be published"); - self.fail_dkg_round(e3_id.clone(), ec, "C2a signing error"); - return; - }; - - // Sign C2b (ESmShareComputation) - let Some(signed_c2b) = self.sign_proof( - e3_id, - ProofType::C2bESmShareComputation, - pending.e_sm_share_computation_proof.expect("checked"), - ) else { - error!("Failed to sign C2b proof — shares will not be published"); - self.fail_dkg_round(e3_id.clone(), ec, "C2b signing error"); - return; - }; - - let Some(signed_c3a_map) = self.sign_and_group_proofs( - e3_id, - ProofType::C3aSkShareEncryption, - pending - .sk_share_encryption_proofs - .iter() - .map(|((recipient, _row), proof)| (*recipient, proof.clone())), - ) else { - error!("Failed to sign C3a proofs — shares will not be published"); - self.fail_dkg_round(e3_id.clone(), ec, "C3a signing error"); - return; - }; - - let Some(signed_c3b_map) = self.sign_and_group_proofs( - e3_id, - ProofType::C3bESmShareEncryption, - pending - .e_sm_share_encryption_proofs - .iter() - .map(|((_esi, recipient, _row), proof)| (*recipient, proof.clone())), - ) else { - error!("Failed to sign C3b proofs — shares will not be published"); - self.fail_dkg_round(e3_id.clone(), ec, "C3b signing error"); - return; - }; - - info!( - "All proofs signed for E3 {} party {} (signer: {})", - e3_id, - party_id, - self.signer.address() - ); - - // Publish local proof events for the node's own state tracking - if let Err(err) = self.bus.publish( - PkGenerationProofSigned { - e3_id: e3_id.clone(), - party_id, - signed_proof: signed_pk_gen, - }, - ec.clone(), - ) { - error!("Failed to publish PkGenerationProofSigned: {err}"); - } - - if let Err(err) = self.bus.publish( - DkgProofSigned { - e3_id: e3_id.clone(), - party_id, - signed_proof: signed_c2a.clone(), - }, - ec.clone(), - ) { - error!("Failed to publish SkDkgProofSigned: {err}"); - } - - if let Err(err) = self.bus.publish( - DkgProofSigned { - e3_id: e3_id.clone(), - party_id, - signed_proof: signed_c2b.clone(), - }, - ec.clone(), - ) { - error!("Failed to publish ESmDkgProofSigned: {err}"); - } - - // Publish C3a signed proofs (reuse already-signed proofs from signed_c3a_map) - for signed_proofs in signed_c3a_map.values() { - for signed in signed_proofs { - if let Err(err) = self.bus.publish( - DkgProofSigned { - e3_id: e3_id.clone(), - party_id, - signed_proof: signed.clone(), - }, - ec.clone(), - ) { - error!("Failed to publish SkShareEncryptionProofSigned: {err}"); - } - } - } - - // Publish C3b signed proofs (reuse already-signed proofs from signed_c3b_map) - for signed_proofs in signed_c3b_map.values() { - for signed in signed_proofs { - if let Err(err) = self.bus.publish( - DkgProofSigned { - e3_id: e3_id.clone(), - party_id, - signed_proof: signed.clone(), - }, - ec.clone(), - ) { - error!("Failed to publish ESmShareEncryptionProofSigned: {err}"); - } - } - } - - // Publish ThresholdShareCreated with proofs attached for each recipient - let share = &pending.full_share; - let num_parties = share.num_parties(); - - info!( - "Publishing ThresholdShareCreated for E3 {} to {} parties", - e3_id, num_parties - ); - - for (positional_idx, &real_party_id) in pending.recipient_party_ids.iter().enumerate() { - match share.extract_for_party(positional_idx) { - Some(party_share) => { - let proof_key = real_party_id as usize; - let c3a_proofs = signed_c3a_map.get(&proof_key).cloned().unwrap_or_default(); - let c3b_proofs = signed_c3b_map.get(&proof_key).cloned().unwrap_or_default(); - - if let Err(err) = self.bus.publish( - ThresholdShareCreated { - e3_id: e3_id.clone(), - share: Arc::new(party_share), - target_party_id: real_party_id, - external: false, - signed_c2a_proof: Some(signed_c2a.clone()), - signed_c2b_proof: Some(signed_c2b.clone()), - signed_c3a_proofs: c3a_proofs, - signed_c3b_proofs: c3b_proofs, - }, - ec.clone(), - ) { - error!( - "Failed to publish ThresholdShareCreated for party {} (idx {}): {err}", - real_party_id, positional_idx - ); - } - } - None if real_party_id == party_id => { - // Own slot is sparse (no self-encryption); nothing to publish. - trace!( - "Skipping ThresholdShareCreated for own slot (party {} idx {})", - real_party_id, - positional_idx - ); - } - None => { - error!( - "Missing encrypted share for recipient party {} (idx {}) from sender party {}; ThresholdShareCreated will not be published for that recipient", - real_party_id, positional_idx, party_id - ); - } - } - } - } - - fn handle_pk_bfv_response( - &mut self, - correlation_id: &CorrelationId, - proof: Proof, - ec: &EventContext, - ) { - let Some(pending) = self.pending.remove(correlation_id) else { - error!( - "Received PkBfv ComputeResponse with correlation_id {:?} but no matching pending request found.", - correlation_id - ); - return; - }; - - let e3_id = pending.e3_id.clone(); - - let mut key = (*pending.key).clone(); - key.proof = Some(proof.clone()); - - // Always sign the proof payload — unsigned proofs are not published - let payload = ProofPayload { - e3_id: e3_id.clone(), - proof_type: ProofType::C0PkBfv, - proof: proof.clone(), - }; - - match SignedProofPayload::sign(payload, &self.signer) { - Ok(signed) => { - info!( - "Signed C0 proof for party {} (signer: {})", - key.party_id, - self.signer.address() - ); - key.signed_payload = Some(signed); - } - Err(err) => { - error!("Failed to sign C0 proof payload: {err} — proof will not be published"); - self.fail_dkg_round(e3_id, ec, "C0 signing error"); - return; - } - } - - let local_party_id = key.party_id; - if let Err(err) = self.bus.publish( - EncryptionKeyCreated { - e3_id: e3_id.clone(), - key: Arc::new(key), - external: false, - }, - ec.clone(), - ) { - error!("Failed to publish EncryptionKeyCreated: {err}"); - } - - // Publish the local node's own C0 as ProofVerificationPassed so the - // CommitmentConsistencyChecker caches it. Without this, C3 proofs from - // other parties that encrypt under this node's pk would fail the C3→C0 - // consistency check (the local C0 target wouldn't exist in the cache). - { - let msg = ( - Bytes::copy_from_slice(&proof.data), - Bytes::copy_from_slice(&proof.public_signals), - ) - .abi_encode(); - let data_hash: [u8; 32] = keccak256(&msg).into(); - - if let Err(err) = self.bus.publish( - ProofVerificationPassed { - e3_id: e3_id.clone(), - party_id: local_party_id, - address: self.signer.address(), - proof_type: ProofType::C0PkBfv, - data_hash, - public_signals: proof.public_signals.clone(), - proof_data: proof.data.clone(), - }, - ec.clone(), - ) { - error!("Failed to publish local C0 ProofVerificationPassed: {err}"); - } - } - - // Emit DKGInnerProofReady for C0, or buffer if meta not yet available - if let Some(meta) = self.node_agg_meta.get(&e3_id) { - if meta.proof_aggregation_enabled { - if let Err(err) = self.bus.publish( - DKGInnerProofReady { - e3_id: e3_id.clone(), - party_id: meta.party_id, - proof: proof.clone(), - seq: 0, - }, - ec.clone(), - ) { - error!("Failed to publish DKGInnerProofReady for C0: {err}"); - } - } - } else { - // ThresholdSharePending hasn't arrived yet — buffer C0 proof - self.node_agg_meta.insert( - e3_id.clone(), - NodeAggregationMeta { - party_id: 0, - total_expected: 0, - pending_c0: Some(proof), - proof_aggregation_enabled: true, // will be overwritten by ThresholdSharePending - }, - ); - } - } - - fn handle_compute_request_error(&mut self, msg: TypedEvent) { - let (msg, ec) = msg.into_components(); - let ComputeRequestErrorKind::Zk(err) = msg.get_err() else { - return; - }; - - if let Some(pending) = self.pending.remove(msg.correlation_id()) { - error!( - "C0 proof request failed for E3 {}: {err} — key will not be published without proof", - pending.e3_id - ); - self.fail_dkg_round(pending.e3_id, &ec, "C0 proof request error"); - return; - } - - if let Some((e3_id, kind, _seq)) = self.threshold_correlation.remove(msg.correlation_id()) { - error!( - "DKG {:?} proof request failed for E3 {}: {err} — threshold share will not be published without proof", - kind, e3_id - ); - self.threshold_correlation - .retain(|_, (eid, _, _)| *eid != e3_id); - self.pending_threshold.remove(&e3_id); - self.fail_dkg_round(e3_id, &ec, "DKG threshold proof request error"); - return; - } - - if let Some((e3_id, kind, _seq)) = self.decryption_correlation.remove(msg.correlation_id()) - { - error!( - "C4 {:?} proof request failed for E3 {}: {err} — DecryptionKeyShared will not be published", - kind, e3_id - ); - self.decryption_correlation - .retain(|_, (eid, _, _)| *eid != e3_id); - self.pending_decryption.remove(&e3_id); - self.fail_dkg_round(e3_id, &ec, "C4 proof request error"); - return; - } - - if let Some(e3_id) = self - .share_decryption_correlation - .remove(msg.correlation_id()) - { - error!( - "C6 proof request failed for E3 {}: {err} — DecryptionshareCreated will not be published", - e3_id - ); - self.pending_share_decryption.remove(&e3_id); - self.fail_decryption_round(e3_id, &ec, "C6 proof request error"); - return; - } - - if let Some(e3_id) = self.pk_aggregation_correlation.remove(msg.correlation_id()) { - error!( - "C5 proof request failed for E3 {}: {err} — PkAggregationProofSigned will not be published", - e3_id - ); - self.pending_pk_aggregation.remove(&e3_id); - - self.fail_dkg_round(e3_id, &ec, "C5 proof request error"); - return; - } - - if let Some(e3_id) = self.aggregation_correlation.remove(msg.correlation_id()) { - error!( - "C7 proof request failed for E3 {}: {err} — AggregationProofSigned will not be published", - e3_id - ); - self.pending_aggregation.remove(&e3_id); - self.fail_decryption_round(e3_id, &ec, "C7 proof request error"); - } - } -} - -impl Actor for ProofRequestActor { - type Context = Context; -} - -impl Handler for ProofRequestActor { - type Result = (); - - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - - match msg { - InterfoldEventData::EncryptionKeyPending(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ThresholdSharePending(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ComputeResponse(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ComputeRequestError(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::DecryptionShareProofsPending(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ShareDecryptionProofPending(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::PkAggregationProofPending(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::AggregationProofPending(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - _ => (), - } - } -} - -impl Handler> for ProofRequestActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_encryption_key_pending(msg) - } -} - -impl Handler> for ProofRequestActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_threshold_share_pending(msg); - } -} - -impl Handler> for ProofRequestActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_compute_response(msg) - } -} - -impl Handler> for ProofRequestActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_compute_request_error(msg) - } -} - -impl Handler> for ProofRequestActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_decryption_share_proofs_pending(msg) - } -} - -impl Handler> for ProofRequestActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_share_decryption_proof_pending(msg) - } -} - -impl Handler> for ProofRequestActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_pk_aggregation_proof_pending(msg) - } -} - -impl Handler> for ProofRequestActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_aggregation_proof_pending(msg) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use alloy::signers::local::PrivateKeySigner; - use anyhow::Result; - use e3_events::{ - ComputeRequestErrorKind, EncryptionKey, Event, HistoryCollector, TakeEvents, Unsequenced, - ZkError, - }; - use e3_test_helpers::get_common_setup; - use e3_utils::utility_types::ArcBytes; - - fn test_ctx(data: impl Into) -> EventContext { - EventContext::::from(data.into()).sequence(0) - } - - async fn next_event( - history: &Addr>, - ) -> Result { - let mut result = history.send(TakeEvents::::new(1)).await?; - assert!(!result.timed_out, "timed out waiting for an event"); - Ok(result.events.pop().expect("expected one event")) - } - - #[actix::test] - async fn c0_compute_error_emits_e3_failed() -> Result<()> { - let (bus, _rng, _seed, _params, _crp, _errors, history) = get_common_setup(None)?; - let mut actor = ProofRequestActor::new(&bus, PrivateKeySigner::random()); - let e3_id = E3id::new("44", 1); - let correlation_id = CorrelationId::new(); - - actor.pending.insert( - correlation_id, - PendingProofRequest { - e3_id: e3_id.clone(), - key: Arc::new(EncryptionKey::new(7, ArcBytes::from_bytes(&[1]))), - }, - ); - - actor.handle_compute_request_error(TypedEvent::new( - ComputeRequestError::new( - ComputeRequestErrorKind::Zk(ZkError::ProofGenerationFailed("boom".to_string())), - ComputeRequest::zk( - ZkRequest::PkBfv(PkBfvProofRequest::new( - ArcBytes::from_bytes(&[1]), - e3_fhe_params::BfvPreset::InsecureThreshold512, - e3_zk_helpers::CiphernodesCommitteeSize::Minimum, - )), - correlation_id, - e3_id.clone(), - ), - ), - test_ctx(E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CommitteeFinalized, - reason: FailureReason::DKGInvalidShares, - }), - )); - - let event = next_event(&history).await?; - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == e3_id - && data.failed_at_stage == E3Stage::CommitteeFinalized - && data.reason == FailureReason::DKGInvalidShares - )); - assert!(actor.pending.is_empty()); - - Ok(()) - } - - #[actix::test] - async fn decryption_failure_helper_emits_e3_failed() -> Result<()> { - let (bus, _rng, _seed, _params, _crp, _errors, history) = get_common_setup(None)?; - let actor = ProofRequestActor::new(&bus, PrivateKeySigner::random()); - let e3_id = E3id::new("45", 1); - - actor.fail_decryption_round( - e3_id.clone(), - &test_ctx(E3Failed { - e3_id: e3_id.clone(), - failed_at_stage: E3Stage::CiphertextReady, - reason: FailureReason::DecryptionInvalidShares, - }), - "test decryption failure", - ); - - let event = next_event(&history).await?; - assert!(matches!( - event.into_data(), - InterfoldEventData::E3Failed(data) - if data.e3_id == e3_id - && data.failed_at_stage == E3Stage::CiphertextReady - && data.reason == FailureReason::DecryptionInvalidShares - )); - - Ok(()) - } -} diff --git a/crates/zk-prover/src/actors/proof_verification.rs b/crates/zk-prover/src/actors/proof_verification.rs deleted file mode 100644 index 060268e58d..0000000000 --- a/crates/zk-prover/src/actors/proof_verification.rs +++ /dev/null @@ -1,618 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Verifies `EncryptionKeyReceived` events: recovers ECDSA address, delegates -//! ZK proof to `ZkActor`, and on failure emits [`SignedProofFailed`] for -//! on-chain fault attribution. - -use std::collections::HashMap; -use std::sync::Arc; - -use actix::{Actor, Addr, AsyncContext, Context, Handler, Message, Recipient}; -use alloy::primitives::{keccak256, Address, Bytes}; -use alloy::sol_types::SolValue; -use e3_events::{ - BusHandle, Committee, E3id, EncryptionKey, EncryptionKeyCreated, EncryptionKeyReceived, - EventContext, EventPublisher, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, - Proof, ProofType, ProofVerificationFailed, ProofVerificationPassed, Sequenced, - SignedProofFailed, SignedProofPayload, TypedEvent, -}; -use e3_fhe_params::BfvPreset; -use e3_request::E3Meta; -use e3_utils::NotifySync; -use e3_zk_helpers::{compute_dkg_pk_commitment_from_public_key_bytes, CiphernodesCommitteeSize}; -use tracing::{error, info, warn}; - -use crate::domain::proof_verification::{validate_external_key, validate_external_key_commitment}; - -#[derive(Debug, Message)] -#[rtype(result = "()")] -pub struct ZkVerificationRequest { - pub proof: Proof, - pub e3_id: E3id, - pub key: Arc, - pub sender: Recipient>, - pub artifacts_dir: String, -} - -#[derive(Debug, Clone, Message)] -#[rtype(result = "()")] -pub struct ZkVerificationResponse { - pub verified: bool, - pub error: Option, - pub e3_id: E3id, - pub key: Arc, -} - -#[derive(Clone, Debug)] -struct PendingVerification { - signed_payload: SignedProofPayload, - recovered_signer: Address, -} - -pub struct ProofVerificationActor { - bus: BusHandle, - verifier: Recipient>, - pending: HashMap<(E3id, u64), PendingVerification>, - /// Tracks preset + committee per E3 so we can derive `artifacts_dir` for proof verification. - presets: HashMap, - /// Canonical finalized committee in party-id order. A C0 signer must own the party slot whose - /// BFV key it advertises; recovering any valid ECDSA address is not sufficient. - committees: HashMap>, -} - -impl ProofVerificationActor { - pub fn new( - bus: &BusHandle, - verifier: Recipient>, - persisted_committees: HashMap, - persisted_e3_metadata: HashMap, - ) -> Self { - let mut actor = Self { - bus: bus.clone(), - verifier, - pending: HashMap::new(), - presets: HashMap::new(), - committees: HashMap::new(), - }; - for (e3_id, meta) in persisted_e3_metadata { - actor.store_preset( - e3_id, - meta.params_preset, - meta.threshold_m, - meta.threshold_n, - ); - } - for (e3_id, committee) in persisted_committees { - actor.store_committee(e3_id, committee.members()); - } - actor - } - - pub fn setup( - bus: &BusHandle, - verifier: Recipient>, - persisted_committees: HashMap, - persisted_e3_metadata: HashMap, - ) -> Addr { - let addr = Self::new(bus, verifier, persisted_committees, persisted_e3_metadata).start(); - bus.subscribe(EventType::CiphernodeSelected, addr.clone().into()); - bus.subscribe(EventType::CommitteeFinalized, addr.clone().into()); - bus.subscribe(EventType::EncryptionKeyReceived, addr.clone().into()); - bus.subscribe(EventType::E3RequestComplete, addr.clone().into()); - addr - } - - fn store_preset( - &mut self, - e3_id: E3id, - preset: BfvPreset, - threshold_m: usize, - threshold_n: usize, - ) { - match CiphernodesCommitteeSize::from_threshold(threshold_m, threshold_n) { - Ok(committee) => { - self.presets.insert(e3_id, (preset, committee)); - } - Err(error) => { - error!( - %e3_id, - threshold_m, - threshold_n, - %error, - "ProofVerificationActor: unrecognised committee; C0 keys will be rejected" - ); - } - } - } - - fn store_committee(&mut self, e3_id: E3id, members: &[String]) { - match members - .iter() - .map(|node| node.parse()) - .collect::, _>>() - { - Ok(committee) => { - self.committees.insert(e3_id, committee); - } - Err(error) => { - error!( - %e3_id, - %error, - "Finalized committee contains an invalid address; C0 keys will be rejected" - ); - } - } - } - - fn handle_encryption_key_received( - &mut self, - msg: TypedEvent, - ctx: &Context, - ) { - let (msg, ec) = msg.into_components(); - let pending_key = (msg.e3_id.clone(), msg.key.party_id); - if self.pending.contains_key(&pending_key) { - warn!( - e3_id = %msg.e3_id, - party_id = msg.key.party_id, - "C0 verification is already pending for party — ignoring duplicate" - ); - return; - } - - let Some((preset, committee_size)) = self.presets.get(&msg.e3_id).copied() else { - error!( - "No BfvPreset known for e3_id={} — cannot determine circuit artifacts directory. \ - This can happen if CiphernodeSelected was missed (e.g. after restart). Rejecting key from party {}.", - msg.e3_id, msg.key.party_id - ); - return; - }; - let Some(expected_signer) = self - .committees - .get(&msg.e3_id) - .and_then(|committee| { - usize::try_from(msg.key.party_id) - .ok() - .and_then(|i| committee.get(i)) - }) - .copied() - else { - error!( - e3_id = %msg.e3_id, - party_id = msg.key.party_id, - "No finalized committee member for C0 party slot — rejecting key" - ); - return; - }; - let validated = match validate_external_key( - &msg.e3_id, - &expected_signer, - msg.key.party_id, - msg.key.proof.as_ref(), - msg.key.signed_payload.as_ref(), - ) { - Ok(validated) => validated, - Err(reason) => { - error!("{reason}"); - return; - } - }; - let proof = msg - .key - .proof - .clone() - .expect("proof present after validation"); - let key_commitment = - match compute_dkg_pk_commitment_from_public_key_bytes(&msg.key.pk_bfv, preset) { - Ok(commitment) => commitment, - Err(err) => { - error!( - e3_id = %msg.e3_id, - party_id = msg.key.party_id, - "Could not bind C0 proof to advertised BFV key: {err} — rejecting key" - ); - return; - } - }; - if let Err(reason) = - validate_external_key_commitment(msg.key.party_id, &proof, &key_commitment) - { - error!("{reason}"); - return; - } - - // Store the signed payload so we can reference it in the verification response - self.pending.insert( - pending_key, - PendingVerification { - signed_payload: validated.signed_payload, - recovered_signer: validated.recovered_signer, - }, - ); - - let artifacts_dir = preset.artifacts_dir_for_committee(committee_size.as_str()); - - let request = TypedEvent::new( - ZkVerificationRequest { - proof: proof.clone(), - e3_id: msg.e3_id, - key: msg.key, - sender: ctx.address().recipient(), - artifacts_dir, - }, - ec, - ); - - self.verifier.do_send(request); - } - - fn publish_key_created( - &self, - e3_id: E3id, - key: Arc, - ec: EventContext, - ) { - if let Err(err) = self.bus.publish( - EncryptionKeyCreated { - e3_id, - key, - external: true, - }, - ec, - ) { - error!("Failed to publish EncryptionKeyCreated: {err}"); - } - } -} - -impl Actor for ProofVerificationActor { - type Context = Context; -} - -impl Handler for ProofVerificationActor { - type Result = (); - - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - match msg { - InterfoldEventData::CiphernodeSelected(data) => { - self.store_preset( - data.e3_id, - data.params_preset, - data.threshold_m, - data.threshold_n, - ); - } - InterfoldEventData::CommitteeFinalized(mut data) => { - // The EVM decoder already emits canonical address order, but sorting again keeps - // this trust boundary correct for replayed/test-produced events as well. - data.sort_by_score(); - self.store_committee(data.e3_id, &data.committee); - } - InterfoldEventData::EncryptionKeyReceived(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::E3RequestComplete(data) => { - let e3_id = data.e3_id; - self.presets.remove(&e3_id); - self.committees.remove(&e3_id); - self.pending - .retain(|(pending_e3, _), _| pending_e3 != &e3_id); - } - _ => (), - } - } -} - -impl Handler> for ProofVerificationActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_encryption_key_received(msg, ctx) - } -} - -impl Handler> for ProofVerificationActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - let (msg, ec) = msg.into_components(); - let pending_key = (msg.e3_id.clone(), msg.key.party_id); - let pending = self.pending.remove(&pending_key); - - if msg.verified { - let Some(PendingVerification { - signed_payload, - recovered_signer, - }) = pending - else { - warn!( - "No pending verification for verified party {} — ignoring duplicate response", - msg.key.party_id - ); - return; - }; - - info!( - "C0 proof verified for party {} - accepting key", - msg.key.party_id - ); - let party_id = msg.key.party_id; - let e3_id = msg.e3_id.clone(); - self.publish_key_created(msg.e3_id, msg.key, ec.clone()); - - // Emit ProofVerificationPassed so AccusationManager can cache success - { - let data_hash: [u8; 32] = { - let msg = ( - Bytes::copy_from_slice(&signed_payload.payload.proof.data), - Bytes::copy_from_slice(&signed_payload.payload.proof.public_signals), - ) - .abi_encode(); - keccak256(&msg).into() - }; - if let Err(err) = self.bus.publish( - ProofVerificationPassed { - e3_id, - party_id, - address: recovered_signer, - proof_type: ProofType::C0PkBfv, - data_hash, - public_signals: signed_payload.payload.proof.public_signals.clone(), - proof_data: signed_payload.payload.proof.data.clone(), - }, - ec, - ) { - error!("Failed to publish ProofVerificationPassed: {err}"); - } - } - } else { - let error_msg = msg.error.unwrap_or_else(|| "unknown error".to_string()); - error!( - "C0 proof verification FAILED for party {} - rejecting key and stopping E3: {}", - msg.key.party_id, error_msg - ); - - if let Some(PendingVerification { - signed_payload, - recovered_signer, - }) = pending - { - warn!( - "Emitting SignedProofFailed for party {} (address: {recovered_signer})", - msg.key.party_id - ); - if let Err(err) = self.bus.publish( - SignedProofFailed { - e3_id: msg.e3_id.clone(), - faulting_node: recovered_signer, - proof_type: signed_payload.payload.proof_type, - signed_payload: signed_payload.clone(), - }, - ec.clone(), - ) { - error!("Failed to publish SignedProofFailed: {err}"); - } - - // Emit ProofVerificationFailed for AccusationManager - let data_hash: [u8; 32] = { - let msg = ( - Bytes::copy_from_slice(&signed_payload.payload.proof.data), - Bytes::copy_from_slice(&signed_payload.payload.proof.public_signals), - ) - .abi_encode(); - keccak256(&msg).into() - }; - if let Err(err) = self.bus.publish( - ProofVerificationFailed { - e3_id: msg.e3_id.clone(), - accused_party_id: msg.key.party_id, - accused_address: recovered_signer, - proof_type: ProofType::C0PkBfv, - data_hash, - signed_payload, - }, - ec.clone(), - ) { - error!("Failed to publish ProofVerificationFailed: {err}"); - } - } - - // NOTE: We do NOT emit E3Failed here. The on-chain SlashingManager - // will expel the faulting node and check if the committee drops below - // threshold. If it does, the contract emits E3Failed on-chain, which - // the EVM reader picks up and propagates to all actors. If the committee - // is still above threshold, the DKG continues with N-1 nodes. - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use actix::{Actor, Context, Handler}; - use alloy::signers::local::PrivateKeySigner; - use e3_events::{ - hlc_factory::HlcFactory, EventBus, EventBusBarrier, EventBusConfig, EventPublisher, - ProofPayload, Seed, Sequencer, StoreEventRequested, StoreEventResponse, - }; - use e3_fhe_params::build_pair_for_preset; - use e3_utils::utility_types::ArcBytes; - use fhe::bfv::{PublicKey, SecretKey}; - use fhe_traits::Serialize; - use tokio::sync::mpsc; - - #[derive(Default)] - struct TestEventStore { - next_seq: u64, - } - - impl Actor for TestEventStore { - type Context = Context; - } - - impl Handler for TestEventStore { - type Result = (); - - fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) { - let StoreEventRequested { event, sender } = msg; - let seq = self.next_seq; - self.next_seq += 1; - sender.do_send(StoreEventResponse(event.into_sequenced(seq))); - } - } - - #[derive(Debug, PartialEq, Eq)] - struct ObservedVerification { - e3_id: E3id, - party_id: u64, - artifacts_dir: String, - } - - struct VerificationRecorder { - sender: mpsc::UnboundedSender, - } - - impl Actor for VerificationRecorder { - type Context = Context; - } - - impl Handler> for VerificationRecorder { - type Result = (); - - fn handle(&mut self, request: TypedEvent, _: &mut Self::Context) { - let (request, _) = request.into_components(); - let _ = self.sender.send(ObservedVerification { - e3_id: request.e3_id, - party_id: request.key.party_id, - artifacts_dir: request.artifacts_dir, - }); - } - } - - fn test_bus() -> BusHandle { - let event_bus = - EventBus::::new(EventBusConfig { deduplicate: true }).start(); - let store = TestEventStore::default().start(); - let sequencer = Sequencer::new(&event_bus, store.recipient()).start(); - BusHandle::new(event_bus, sequencer, HlcFactory::new()).enable("c0-recovery-test") - } - - fn signed_c0_key( - signer: &PrivateKeySigner, - party_id: u64, - e3_id: &E3id, - preset: BfvPreset, - ) -> Arc { - let (_, dkg_params) = build_pair_for_preset(preset).expect("build BFV test parameters"); - let mut rng = rand::rng(); - let secret_key = SecretKey::random(&dkg_params, &mut rng); - let public_key = PublicKey::new(&secret_key, &mut rng); - let public_key_bytes = public_key.to_bytes(); - let commitment = compute_dkg_pk_commitment_from_public_key_bytes(&public_key_bytes, preset) - .expect("compute C0 key commitment"); - let proof = Proof::new( - ProofType::C0PkBfv.circuit_names()[0], - ArcBytes::from_bytes(&[1, 2, 3]), - ArcBytes::from_bytes(&commitment), - ); - let signed = SignedProofPayload::sign( - ProofPayload { - e3_id: e3_id.clone(), - proof_type: ProofType::C0PkBfv, - proof: proof.clone(), - }, - signer, - ) - .expect("sign C0 test proof"); - - Arc::new( - EncryptionKey::new(party_id, ArcBytes::from_bytes(&public_key_bytes)) - .with_proof(proof) - .with_signed_payload(signed), - ) - } - - #[actix::test] - async fn restored_context_dispatches_c0_without_replayed_lifecycle_events() { - let bus = test_bus(); - let e3_id = E3id::new("7", 31_337); - let preset = BfvPreset::InsecureThreshold512; - let signer = PrivateKeySigner::random(); - let mut committee_members = vec![ - signer.address().to_string(), - PrivateKeySigner::random().address().to_string(), - PrivateKeySigner::random().address().to_string(), - ]; - committee_members.sort_by_key(|member| member.to_lowercase()); - let party_id = committee_members - .iter() - .position(|member| member.eq_ignore_ascii_case(&signer.address().to_string())) - .expect("signer belongs to restored committee") as u64; - - let persisted_committees = - HashMap::from([(e3_id.clone(), Committee::new(committee_members))]); - let persisted_e3_metadata = HashMap::from([( - e3_id.clone(), - E3Meta { - threshold_m: 1, - threshold_n: 3, - seed: Seed([0; 32]), - params_preset: preset, - params: ArcBytes::default(), - error_size: ArcBytes::default(), - proof_aggregation_enabled: false, - }, - )]); - let (observed_tx, mut observed_rx) = mpsc::unbounded_channel(); - let verifier = VerificationRecorder { - sender: observed_tx, - } - .start(); - ProofVerificationActor::setup( - &bus, - verifier.recipient(), - persisted_committees, - persisted_e3_metadata, - ); - - // No CiphernodeSelected or CommitteeFinalized event is published: both are already - // represented by durable snapshots and may be outside the post-snapshot replay range. - bus.event_bus() - .send(EventBusBarrier) - .await - .expect("event bus barrier"); - bus.publish_without_context(EncryptionKeyReceived { - e3_id: e3_id.clone(), - key: signed_c0_key(&signer, party_id, &e3_id, preset), - }) - .expect("publish external C0 key"); - - let observed = tokio::time::timeout(std::time::Duration::from_secs(2), observed_rx.recv()) - .await - .expect("restored C0 context did not dispatch verification before timeout") - .expect("verification recorder stopped"); - assert_eq!( - observed, - ObservedVerification { - e3_id, - party_id, - artifacts_dir: preset - .artifacts_dir_for_committee(CiphernodesCommitteeSize::Minimum.as_str()), - } - ); - } -} diff --git a/crates/zk-prover/src/actors/share_verification.rs b/crates/zk-prover/src/actors/share_verification.rs deleted file mode 100644 index 1027af0ca7..0000000000 --- a/crates/zk-prover/src/actors/share_verification.rs +++ /dev/null @@ -1,810 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Actor for C2/C3/C4 share proof verification. -//! -//! Follows the same pattern as [`ProofVerificationActor`] (for C0) — sits -//! between the raw proof data and the verified result, handling ECDSA validation -//! and ZK verification orchestration. -//! -//! ## Flow -//! -//! 1. Receives [`ShareVerificationDispatched`] from [`ThresholdKeyshare`]. -//! 2. Performs ECDSA validation (signature recovery, signer consistency, e3_id, -//! circuit name) — lightweight, no thread pool needed. -//! 3. Dispatches ZK-only verification to multithread via [`ComputeRequest`]. -//! 4. Receives [`ComputeResponse`] from multithread with pure ZK results. -//! 5. Combines ECDSA + ZK results. -//! 6. Emits [`SignedProofFailed`] for any failing proofs. -//! 7. Publishes [`ShareVerificationComplete`] with dishonest party set. - -use std::collections::{BTreeSet, HashMap, HashSet}; - -use actix::{Actor, Addr, Context, Handler}; -use alloy::primitives::{keccak256, Address, Bytes}; -use alloy::sol_types::SolValue; -use e3_events::{ - BusHandle, CommitmentConsistencyCheckComplete, CommitmentConsistencyCheckRequested, Committee, - ComputeRequest, ComputeRequestError, ComputeResponse, ComputeResponseKind, CorrelationId, E3id, - EventContext, EventPublisher, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, - PartyVerificationResult, ProofType, ProofVerificationFailed, ProofVerificationPassed, - Sequenced, ShareVerificationComplete, ShareVerificationDispatched, SignedProofFailed, - SignedProofPayload, TypedEvent, VerificationKind, VerifyShareDecryptionProofsRequest, - VerifyShareProofsRequest, ZkRequest, ZkResponse, -}; -use e3_utils::utility_types::ArcBytes; -use e3_utils::NotifySync; -use tracing::{error, info, warn}; - -use crate::domain::share_verification::{ - filter_consistent, label_for, PendingConsistencyCheck, PendingVerification, ShareVerifier, - VerifiableParty, ZkPartyEmission, -}; - -/// Actor that handles C1/C2/C3/C4/C6 share proof verification. -/// -/// Three-stage pipeline: -/// 1. ECDSA validation (lightweight, done inline) -/// 2. Commitment consistency check (dispatched to per-E3 checker via event bus) -/// 3. ZK proof verification (heavyweight, delegated to multithread) -/// -/// Emits [`SignedProofFailed`] for fault attribution and -/// [`ShareVerificationComplete`] with the final dishonest party set. -pub struct ShareVerificationActor { - bus: BusHandle, - /// Canonical finalized committees in party-id order. Every signed C1-C4/C6 proof must recover - /// to the address that owns its outer `sender_party_id` slot. - committees: HashMap>, - /// Tracks pending ZK verifications by correlation ID. - pending: HashMap, - /// Tracks pending consistency checks by correlation ID (between ECDSA and ZK). - pending_consistency: HashMap, -} - -impl ShareVerificationActor { - pub fn new(bus: &BusHandle, persisted_committees: HashMap) -> Self { - let mut actor = Self { - bus: bus.clone(), - committees: HashMap::new(), - pending: HashMap::new(), - pending_consistency: HashMap::new(), - }; - for (e3_id, committee) in persisted_committees { - actor.store_committee(e3_id, committee.members()); - } - actor - } - - pub fn setup(bus: &BusHandle, persisted_committees: HashMap) -> Addr { - let addr = Self::new(bus, persisted_committees).start(); - bus.subscribe(EventType::CommitteeFinalized, addr.clone().into()); - bus.subscribe(EventType::ShareVerificationDispatched, addr.clone().into()); - bus.subscribe(EventType::ComputeResponse, addr.clone().into()); - bus.subscribe(EventType::ComputeRequestError, addr.clone().into()); - bus.subscribe( - EventType::CommitmentConsistencyCheckComplete, - addr.clone().into(), - ); - bus.subscribe(EventType::E3RequestComplete, addr.clone().into()); - addr - } - - fn store_committee(&mut self, e3_id: E3id, members: &[String]) { - match members - .iter() - .map(|member| member.parse()) - .collect::, _>>() - { - Ok(committee) => { - self.committees.insert(e3_id, committee); - } - Err(error) => { - error!( - %e3_id, - %error, - "Finalized committee contains an invalid address; share proofs will be rejected" - ); - } - } - } - - fn handle_share_verification_dispatched( - &mut self, - msg: TypedEvent, - ) { - let (msg, ec) = msg.into_components(); - let e3_id = msg.e3_id.clone(); - - info!( - "handling ShareVerificationDispatched {:?}, {:?}", - e3_id, msg.kind - ); - - let params_preset = msg.params_preset; - let committee_size = msg.committee_size; - match msg.kind { - VerificationKind::ShareProofs - | VerificationKind::ThresholdDecryptionProofs - | VerificationKind::PkGenerationProofs => { - let kind = msg.kind.clone(); - self.verify_proofs( - e3_id, - kind.clone(), - msg.share_proofs, - msg.pre_dishonest, - ec, - params_preset, - committee_size, - |pending, passed| { - pending.ecdsa_passed_share_proofs = passed; - }, - ); - } - VerificationKind::DecryptionProofs => { - self.verify_proofs( - e3_id, - VerificationKind::DecryptionProofs, - msg.decryption_proofs, - msg.pre_dishonest, - ec, - params_preset, - committee_size, - |pending, passed| { - pending.ecdsa_passed_decryption_proofs = passed; - }, - ); - } - } - } - - /// Generic ECDSA validation + consistency check dispatch. - /// - /// After ECDSA validation, publishes [`CommitmentConsistencyCheckRequested`] - /// and stores a [`PendingConsistencyCheck`]. ZK verification is deferred - /// until the consistency check response arrives. - #[allow(clippy::too_many_arguments)] - fn verify_proofs( - &mut self, - e3_id: E3id, - kind: VerificationKind, - party_proofs: Vec

, - pre_dishonest: BTreeSet, - ec: EventContext, - params_preset: e3_fhe_params::BfvPreset, - committee_size: e3_zk_helpers::CiphernodesCommitteeSize, - store_passed_proofs: impl FnOnce(&mut PendingConsistencyCheck, Vec

), - ) { - let e3_id_str = e3_id.to_string(); - let label = label_for(&kind); - - // Pure ECDSA validation + proof-commitment preparation lives in the - // domain service; the actor only emits failures, stores pending state, - // and publishes the consistency-check request. - let committee = self.committees.get(&e3_id).map(Vec::as_slice); - let outcome = ShareVerifier::validate_and_prepare( - &party_proofs, - &e3_id_str, - &kind, - label, - committee, - params_preset, - committee_size, - ); - - for failure in &outcome.failures { - self.emit_signed_proof_failed( - &e3_id, - &failure.signed, - failure.recovered, - failure.party_id, - &ec, - ); - } - - if outcome.ecdsa_passed_parties.is_empty() { - // All parties failed ECDSA — publish result immediately - let mut all_dishonest: BTreeSet = pre_dishonest; - all_dishonest.extend(outcome.ecdsa_dishonest); - self.publish_complete(e3_id, kind, all_dishonest, ec); - return; - } - - // Store pending consistency check with the original party proofs - let correlation_id = CorrelationId::new(); - let mut pending = PendingConsistencyCheck { - e3_id: e3_id.clone(), - kind: kind.clone(), - ec: ec.clone(), - ecdsa_dishonest: outcome.ecdsa_dishonest, - pre_dishonest, - party_addresses: outcome.party_addresses, - party_proof_hashes: outcome.party_proof_hashes, - party_public_signals: outcome.party_public_signals, - party_proof_data: outcome.party_proof_data, - ecdsa_passed_share_proofs: Vec::new(), - ecdsa_passed_decryption_proofs: Vec::new(), - params_preset, - committee_size, - }; - store_passed_proofs(&mut pending, outcome.ecdsa_passed_parties); - self.pending_consistency.insert(correlation_id, pending); - - // Publish consistency check request - if let Err(err) = self.bus.publish( - CommitmentConsistencyCheckRequested { - e3_id: e3_id.clone(), - kind: kind.clone(), - correlation_id, - party_proofs: outcome.consistency_party_data, - }, - ec.clone(), - ) { - error!( - "Failed to dispatch {} consistency check: {err} — treating all as dishonest", - label - ); - if let Some(pending) = self.pending_consistency.remove(&correlation_id) { - let mut all_dishonest: BTreeSet = pending.pre_dishonest; - all_dishonest.extend(pending.ecdsa_dishonest); - for p in &pending.ecdsa_passed_share_proofs { - all_dishonest.insert(p.sender_party_id); - } - for p in &pending.ecdsa_passed_decryption_proofs { - all_dishonest.insert(p.sender_party_id); - } - self.publish_complete(e3_id, kind, all_dishonest, ec); - } - } - } - - /// Handle consistency check response: add inconsistent parties to the - /// dishonest set, then dispatch ZK verification for the remaining - /// consistent parties. - fn handle_consistency_check_complete( - &mut self, - msg: TypedEvent, - ) { - let (data, _ec) = msg.into_components(); - - let Some(pending) = self.pending_consistency.remove(&data.correlation_id) else { - return; // Not our correlation ID - }; - - let label = label_for(&pending.kind); - - if !data.inconsistent_parties.is_empty() { - warn!( - "{} consistency check found {} inconsistent parties for E3 {}: {:?}", - label, - data.inconsistent_parties.len(), - pending.e3_id, - data.inconsistent_parties - ); - } - - // Accumulate all dishonest parties discovered so far - let mut dishonest_so_far: BTreeSet = pending.pre_dishonest.clone(); - dishonest_so_far.extend(&pending.ecdsa_dishonest); - dishonest_so_far.extend(&data.inconsistent_parties); - - // Filter ECDSA-passed proofs to only consistent parties and dispatch ZK - let inconsistent = &data.inconsistent_parties; - let zk_correlation_id = CorrelationId::new(); - - let (request, dispatched_party_ids) = match pending.kind { - VerificationKind::ShareProofs - | VerificationKind::ThresholdDecryptionProofs - | VerificationKind::PkGenerationProofs => { - let Some((passed, ids)) = - filter_consistent(pending.ecdsa_passed_share_proofs, inconsistent, |p| { - p.sender_party_id - }) - else { - self.publish_complete( - pending.e3_id, - pending.kind, - dishonest_so_far, - pending.ec, - ); - return; - }; - let req = ComputeRequest::zk( - ZkRequest::VerifyShareProofs(VerifyShareProofsRequest { - party_proofs: passed, - params_preset: pending.params_preset, - committee_size: pending.committee_size, - }), - zk_correlation_id, - pending.e3_id.clone(), - ); - (req, ids) - } - VerificationKind::DecryptionProofs => { - let Some((passed, ids)) = - filter_consistent(pending.ecdsa_passed_decryption_proofs, inconsistent, |p| { - p.sender_party_id - }) - else { - self.publish_complete( - pending.e3_id, - pending.kind, - dishonest_so_far, - pending.ec, - ); - return; - }; - let req = ComputeRequest::zk( - ZkRequest::VerifyShareDecryptionProofs(VerifyShareDecryptionProofsRequest { - party_proofs: passed, - params_preset: pending.params_preset, - committee_size: pending.committee_size, - }), - zk_correlation_id, - pending.e3_id.clone(), - ); - (req, ids) - } - }; - - // Only keep proof hashes/signals/addresses for parties going to ZK - let party_addresses: HashMap = pending - .party_addresses - .into_iter() - .filter(|(pid, _)| dispatched_party_ids.contains(pid)) - .collect(); - let party_proof_hashes: HashMap> = pending - .party_proof_hashes - .into_iter() - .filter(|(pid, _)| dispatched_party_ids.contains(pid)) - .collect(); - let party_public_signals: HashMap> = pending - .party_public_signals - .into_iter() - .filter(|(pid, _)| dispatched_party_ids.contains(pid)) - .collect(); - let party_proof_data: HashMap> = pending - .party_proof_data - .into_iter() - .filter(|(pid, _)| dispatched_party_ids.contains(pid)) - .collect(); - - // Store pending ZK verification state. - // All prior dishonest parties (pre_dishonest + ECDSA + consistency) are - // folded into `pre_dishonest` so that `handle_compute_response` produces - // the correct final dishonest set when it adds ZK failures. - self.pending.insert( - zk_correlation_id, - PendingVerification { - e3_id: pending.e3_id.clone(), - kind: pending.kind.clone(), - ec: pending.ec.clone(), - ecdsa_dishonest: HashSet::new(), - pre_dishonest: dishonest_so_far, - dispatched_party_ids: dispatched_party_ids.clone(), - party_addresses, - party_proof_hashes, - party_public_signals, - party_proof_data, - params_preset: pending.params_preset, - committee_size: pending.committee_size, - }, - ); - - if let Err(err) = self.bus.publish(request, pending.ec.clone()) { - error!( - "Failed to dispatch {} ZK verification after consistency check: {err}", - label - ); - if let Some(zk_pending) = self.pending.remove(&zk_correlation_id) { - let mut all_dishonest: BTreeSet = zk_pending.pre_dishonest; - all_dishonest.extend(zk_pending.ecdsa_dishonest); - all_dishonest.extend(zk_pending.dispatched_party_ids); - self.publish_complete( - zk_pending.e3_id, - zk_pending.kind, - all_dishonest, - zk_pending.ec, - ); - } - } - } - - /// Handle ZK verification response from multithread. - fn handle_compute_response(&mut self, msg: TypedEvent) { - let (msg, _ec) = msg.into_components(); - - let correlation_id = msg.correlation_id; - let Some(pending) = self.pending.remove(&correlation_id) else { - return; // Not our correlation ID - }; - - let zk_results: Vec = match (&pending.kind, msg.response) { - ( - VerificationKind::ShareProofs - | VerificationKind::ThresholdDecryptionProofs - | VerificationKind::PkGenerationProofs, - ComputeResponseKind::Zk(ZkResponse::VerifyShareProofs(r)), - ) => r.party_results, - ( - VerificationKind::DecryptionProofs, - ComputeResponseKind::Zk(ZkResponse::VerifyShareDecryptionProofs(r)), - ) => r.party_results, - _ => { - error!("Unexpected ComputeResponse kind for verification — treating all dispatched parties as dishonest"); - let mut all_dishonest: BTreeSet = pending.pre_dishonest; - all_dishonest.extend(pending.ecdsa_dishonest); - all_dishonest.extend(pending.dispatched_party_ids); - self.publish_complete(pending.e3_id, pending.kind, all_dishonest, pending.ec); - return; - } - }; - - // Pure tally (dishonest accounting + emission decisions) lives in the - // domain service; the actor performs the resulting bus publishes. - let tally = ShareVerifier::tally_zk_results( - pending.pre_dishonest, - &pending.ecdsa_dishonest, - &pending.dispatched_party_ids, - &zk_results, - ); - - for emission in tally.emissions { - match emission { - ZkPartyEmission::Failed { party_id, signed } => { - let addr = pending.party_addresses.get(&party_id).copied(); - self.emit_signed_proof_failed( - &pending.e3_id, - &signed, - addr, - party_id, - &pending.ec, - ); - } - ZkPartyEmission::Passed { party_id } => { - // Emit ProofVerificationPassed for each proof type from this party - if let Some(hashes) = pending.party_proof_hashes.get(&party_id) { - let addr = pending - .party_addresses - .get(&party_id) - .copied() - .unwrap_or_default(); - let signals = pending.party_public_signals.get(&party_id); - let datas = pending.party_proof_data.get(&party_id); - for (i, &(proof_type, data_hash)) in hashes.iter().enumerate() { - let public_signals = signals - .and_then(|s| s.get(i)) - .map(|(_, ps)| ps.clone()) - .unwrap_or_default(); - let proof_data = datas - .and_then(|d| d.get(i)) - .map(|(_, pd)| pd.clone()) - .unwrap_or_default(); - if let Err(err) = self.bus.publish( - ProofVerificationPassed { - e3_id: pending.e3_id.clone(), - party_id, - address: addr, - proof_type, - data_hash, - public_signals, - proof_data, - }, - pending.ec.clone(), - ) { - error!("Failed to publish ProofVerificationPassed: {err}"); - } - } - } - } - } - } - - self.publish_complete(pending.e3_id, pending.kind, tally.dishonest, pending.ec); - } - - fn emit_signed_proof_failed( - &self, - e3_id: &E3id, - signed_payload: &SignedProofPayload, - recovered_addr: Option

, - party_id: u64, - ec: &EventContext, - ) { - let faulting_node = match recovered_addr { - Some(addr) => addr, - None => match signed_payload.recover_address() { - Ok(addr) => addr, - Err(err) => { - warn!( - "Signature recovery failed for party {} — using zero address for fault attribution: {err}", - party_id - ); - Address::ZERO - } - }, - }; - - if let Err(err) = self.bus.publish( - SignedProofFailed { - e3_id: e3_id.clone(), - faulting_node, - proof_type: signed_payload.payload.proof_type, - signed_payload: signed_payload.clone(), - }, - ec.clone(), - ) { - error!("Failed to publish SignedProofFailed: {err}"); - } - - // Also emit ProofVerificationFailed for AccusationManager - let data_hash: [u8; 32] = { - let msg = ( - Bytes::copy_from_slice(&signed_payload.payload.proof.data), - Bytes::copy_from_slice(&signed_payload.payload.proof.public_signals), - ) - .abi_encode(); - keccak256(&msg).into() - }; - if let Err(err) = self.bus.publish( - ProofVerificationFailed { - e3_id: e3_id.clone(), - accused_party_id: party_id, - accused_address: faulting_node, - proof_type: signed_payload.payload.proof_type, - data_hash, - signed_payload: signed_payload.clone(), - }, - ec.clone(), - ) { - error!("Failed to publish ProofVerificationFailed: {err}"); - } - } - - /// Handle computation error from multithread — clean up pending state and - /// publish ShareVerificationComplete treating all dispatched parties as dishonest. - fn handle_compute_request_error(&mut self, msg: TypedEvent) { - let (msg, _ec) = msg.into_components(); - - let correlation_id = msg.correlation_id(); - let Some(pending) = self.pending.remove(correlation_id) else { - return; - }; - - error!( - "ZK verification computation failed for E3 {} ({:?}): {} — treating all dispatched parties as dishonest", - pending.e3_id, pending.kind, msg - ); - - let mut all_dishonest: BTreeSet = pending.pre_dishonest; - all_dishonest.extend(pending.ecdsa_dishonest); - all_dishonest.extend(pending.dispatched_party_ids); - self.publish_complete(pending.e3_id, pending.kind, all_dishonest, pending.ec); - } - - fn publish_complete( - &self, - e3_id: E3id, - kind: VerificationKind, - dishonest_parties: BTreeSet, - ec: EventContext, - ) { - if let Err(err) = self.bus.publish( - ShareVerificationComplete { - e3_id, - kind, - dishonest_parties, - }, - ec, - ) { - error!("Failed to publish ShareVerificationComplete: {err}"); - } - } -} - -impl Actor for ShareVerificationActor { - type Context = Context; -} - -impl Handler for ShareVerificationActor { - type Result = (); - - fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { - let (msg, ec) = msg.into_components(); - match msg { - InterfoldEventData::CommitteeFinalized(mut data) => { - // Mirror the C0 verifier's canonical ordering at this trust boundary. Replayed and - // test-produced events are not assumed to have passed through the EVM decoder. - data.sort_by_score(); - self.store_committee(data.e3_id, &data.committee); - } - InterfoldEventData::ShareVerificationDispatched(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ComputeResponse(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::ComputeRequestError(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::CommitmentConsistencyCheckComplete(data) => { - self.notify_sync(ctx, TypedEvent::new(data, ec)) - } - InterfoldEventData::E3RequestComplete(data) => { - let e3_id = data.e3_id; - self.committees.remove(&e3_id); - self.pending.retain(|_, pending| pending.e3_id != e3_id); - self.pending_consistency - .retain(|_, pending| pending.e3_id != e3_id); - } - _ => (), - } - } -} - -impl Handler> for ShareVerificationActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_share_verification_dispatched(msg) - } -} - -impl Handler> for ShareVerificationActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_compute_response(msg) - } -} - -impl Handler> for ShareVerificationActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_compute_request_error(msg) - } -} - -impl Handler> for ShareVerificationActor { - type Result = (); - - fn handle( - &mut self, - msg: TypedEvent, - _ctx: &mut Self::Context, - ) -> Self::Result { - self.handle_consistency_check_complete(msg) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use actix::{Actor, Context, Handler}; - use alloy::signers::local::PrivateKeySigner; - use e3_events::{ - hlc_factory::HlcFactory, Event, EventBus, EventBusBarrier, EventBusConfig, EventPublisher, - Proof, ProofPayload, Sequencer, StoreEventRequested, StoreEventResponse, - }; - use e3_fhe_params::BfvPreset; - use e3_utils::utility_types::ArcBytes; - use e3_zk_helpers::CiphernodesCommitteeSize; - use std::{collections::BTreeSet, time::Duration}; - - #[derive(Default)] - struct TestEventStore { - next_seq: u64, - } - - impl Actor for TestEventStore { - type Context = Context; - } - - impl Handler for TestEventStore { - type Result = (); - - fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) { - let StoreEventRequested { event, sender } = msg; - let seq = self.next_seq; - self.next_seq += 1; - sender.do_send(StoreEventResponse(event.into_sequenced(seq))); - } - } - - fn test_bus() -> BusHandle { - let event_bus = - EventBus::::new(EventBusConfig { deduplicate: true }).start(); - let store = TestEventStore::default().start(); - let sequencer = Sequencer::new(&event_bus, store.recipient()).start(); - BusHandle::new(event_bus, sequencer, HlcFactory::new()).enable("share-recovery-test") - } - - fn signed_c6(signer: &PrivateKeySigner, e3_id: &E3id, marker: u8) -> SignedProofPayload { - let proof_type = ProofType::C6ThresholdShareDecryption; - let proof = Proof::new( - proof_type.circuit_names()[0], - ArcBytes::from_bytes(&[marker, 2, 3]), - ArcBytes::from_bytes(&[4, 5, marker]), - ); - SignedProofPayload::sign( - ProofPayload { - e3_id: e3_id.clone(), - proof_type, - proof, - }, - signer, - ) - .expect("sign C6 test proof") - } - - #[actix::test] - async fn restored_committee_authorizes_c6_without_replayed_finalization_event() { - let bus = test_bus(); - let e3_id = E3id::new("0", 31_337); - let signers = [ - PrivateKeySigner::random(), - PrivateKeySigner::random(), - PrivateKeySigner::random(), - ]; - let committee = Committee::new( - signers - .iter() - .map(|signer| signer.address().to_string()) - .collect(), - ); - ShareVerificationActor::setup(&bus, HashMap::from([(e3_id.clone(), committee)])); - - // Register the observer and fence the EventBus before dispatching. No - // CommitteeFinalized event is published: this models restart after that event was - // incorporated into the durable snapshot and excluded from the replay range. - let consistency_requested = bus.wait_for(EventType::CommitmentConsistencyCheckRequested); - bus.event_bus() - .send(EventBusBarrier) - .await - .expect("event bus barrier"); - - bus.publish_without_context(ShareVerificationDispatched { - e3_id: e3_id.clone(), - kind: VerificationKind::ThresholdDecryptionProofs, - share_proofs: signers[..2] - .iter() - .enumerate() - .map(|(party_id, signer)| e3_events::PartyProofsToVerify { - sender_party_id: party_id as u64, - signed_proofs: vec![signed_c6(signer, &e3_id, party_id as u8)], - }) - .collect(), - decryption_proofs: Vec::new(), - pre_dishonest: BTreeSet::new(), - params_preset: BfvPreset::InsecureThreshold512, - committee_size: CiphernodesCommitteeSize::Minimum, - }) - .expect("publish C6 verification dispatch"); - - let event = tokio::time::timeout(Duration::from_secs(2), consistency_requested) - .await - .expect("restored committee did not authorize C6 before timeout") - .expect("wait for consistency request"); - let InterfoldEventData::CommitmentConsistencyCheckRequested(request) = event.into_data() - else { - panic!("unexpected event type") - }; - assert_eq!(request.e3_id, e3_id); - assert_eq!(request.party_proofs.len(), 2); - } -} diff --git a/crates/zk-prover/src/actors/commitment_consistency_checker.rs b/crates/zk-prover/src/commitment_consistency_checker.rs similarity index 100% rename from crates/zk-prover/src/actors/commitment_consistency_checker.rs rename to crates/zk-prover/src/commitment_consistency_checker.rs diff --git a/crates/zk-prover/src/actors/commitment_consistency_checker_ext.rs b/crates/zk-prover/src/commitment_consistency_checker_ext.rs similarity index 100% rename from crates/zk-prover/src/actors/commitment_consistency_checker_ext.rs rename to crates/zk-prover/src/commitment_consistency_checker_ext.rs diff --git a/crates/zk-prover/src/actors/commitment_links/c0_to_c3.rs b/crates/zk-prover/src/commitment_links/c0_to_c3.rs similarity index 100% rename from crates/zk-prover/src/actors/commitment_links/c0_to_c3.rs rename to crates/zk-prover/src/commitment_links/c0_to_c3.rs diff --git a/crates/zk-prover/src/actors/commitment_links/c1_to_c2.rs b/crates/zk-prover/src/commitment_links/c1_to_c2.rs similarity index 100% rename from crates/zk-prover/src/actors/commitment_links/c1_to_c2.rs rename to crates/zk-prover/src/commitment_links/c1_to_c2.rs diff --git a/crates/zk-prover/src/actors/commitment_links/c1_to_c5.rs b/crates/zk-prover/src/commitment_links/c1_to_c5.rs similarity index 100% rename from crates/zk-prover/src/actors/commitment_links/c1_to_c5.rs rename to crates/zk-prover/src/commitment_links/c1_to_c5.rs diff --git a/crates/zk-prover/src/actors/commitment_links/c2_to_c3.rs b/crates/zk-prover/src/commitment_links/c2_to_c3.rs similarity index 100% rename from crates/zk-prover/src/actors/commitment_links/c2_to_c3.rs rename to crates/zk-prover/src/commitment_links/c2_to_c3.rs diff --git a/crates/zk-prover/src/actors/commitment_links/c2_to_c4.rs b/crates/zk-prover/src/commitment_links/c2_to_c4/mod.rs similarity index 55% rename from crates/zk-prover/src/actors/commitment_links/c2_to_c4.rs rename to crates/zk-prover/src/commitment_links/c2_to_c4/mod.rs index c22e1bc486..35e3499779 100644 --- a/crates/zk-prover/src/actors/commitment_links/c2_to_c4.rs +++ b/crates/zk-prover/src/commitment_links/c2_to_c4/mod.rs @@ -230,174 +230,4 @@ fn check_exact_l_commitments( } #[cfg(test)] -mod tests { - use super::*; - - fn make_field(val: u8) -> [u8; 32] { - let mut f = [0u8; 32]; - f[31] = val; - f - } - - /// C2 inner signals: [expected_secret_commitment] + share commitments (row-major: party, mod). - fn c2_signals(share_commits: &[[u8; 32]]) -> Vec { - let mut v = Vec::new(); - v.extend_from_slice(&make_field(0xFF)); // expected_secret_commitment (skipped) - for c in share_commits { - v.extend_from_slice(c); - } - v - } - - /// C4 signals: [expected_commitments row-major (party, mod)..., aggregated_commitment]. - fn c4_signals(rows: &[Vec<[u8; 32]>], aggregated: [u8; 32]) -> Vec { - let mut v = Vec::new(); - for row in rows { - for c in row { - v.extend_from_slice(c); - } - } - v.extend_from_slice(&aggregated); - v - } - - #[test] - fn extract_share_commitments_from_c2() { - let link = C2aToC4aShareCommitmentLink { l: 2 }; - // C2 with 3 parties × 2 moduli = 6 share commits - let commits: Vec<[u8; 32]> = (1u8..=6).map(make_field).collect(); - let c2 = c2_signals(&commits); - let values = link.extract_source_values(&c2); - assert_eq!(values.len(), 6); - assert_eq!(values[0], make_field(1)); - assert_eq!(values[5], make_field(6)); - } - - #[test] - fn extract_skips_secret_commitment() { - let link = C2aToC4aShareCommitmentLink { l: 2 }; - let c2 = c2_signals(&[make_field(1), make_field(2)]); - let values = link.extract_source_values(&c2); - assert_eq!(values.len(), 2); - assert!(!values.contains(&make_field(0xFF))); - } - - /// 3 parties (N=3), 2 moduli (L=2), 2 honest parties (H=2). - /// C2 from party X=1, C4 for recipient R=1. - /// C2_X: [p0m0,p0m1, p1m0,p1m1, p2m0,p2m1] - /// C4_R=1: rows for party 0 and party 1 = [[p0m0,p0m1],[p1m0,p1m1]] + agg - /// check_consistency(src=1, tgt=1): C2 slot R=1 = [p1m0,p1m1] must equal C4 row X=1 = [p1m0,p1m1] - #[test] - fn consistency_passes_precise_l_way_check() { - let l = 2; - let link = C2aToC4aShareCommitmentLink { l }; - - // C2 from sender X=1: 3 parties × 2 moduli - let c2 = c2_signals(&[ - make_field(10), - make_field(11), // party 0 - make_field(20), - make_field(21), // party 1 (slot for tgt_party=1) - make_field(30), - make_field(31), // party 2 - ]); - let source_values = link.extract_source_values(&c2); - - // C4 for recipient R=1: 2 honest parties (rows for X=0 and X=1) - let c4 = c4_signals( - &[ - vec![make_field(10), make_field(11)], // row X=0 - vec![make_field(20), make_field(21)], // row X=1 (sender's commits for this recipient) - ], - make_field(99), // aggregated output - ); - - // src_party_id=1 (sender X=1), tgt_party_id=1 (recipient R=1) - assert!(link.check_consistency(&source_values, &c4, 1, 1)); - } - - #[test] - fn consistency_fails_when_wrong_modulus_commitment() { - let l = 2; - let link = C2aToC4aShareCommitmentLink { l }; - - let c2 = c2_signals(&[ - make_field(10), - make_field(11), - make_field(20), - make_field(21), // party 1 slot - make_field(30), - make_field(31), - ]); - let source_values = link.extract_source_values(&c2); - - // C4 has correct first modulus (20) but wrong second (99 instead of 21) - let c4 = c4_signals( - &[ - vec![make_field(10), make_field(11)], - vec![make_field(20), make_field(99)], // second modulus wrong - ], - make_field(0), - ); - - assert!(!link.check_consistency(&source_values, &c4, 1, 1)); - } - - #[test] - fn consistency_fails_when_wrong_party_slot() { - let l = 2; - let link = C2aToC4aShareCommitmentLink { l }; - - let c2 = c2_signals(&[ - make_field(10), - make_field(11), // party 0 - make_field(20), - make_field(21), // party 1 - ]); - let source_values = link.extract_source_values(&c2); - - // C4 has party-0 commits in row 0 only - let c4 = c4_signals(&[vec![make_field(10), make_field(11)]], make_field(0)); - - // src=0, tgt=1: C2 slot 1 = [20,21], C4 row 0 = [10,11] — mismatch - assert!(!link.check_consistency(&source_values, &c4, 0, 1)); - } - - #[test] - fn consistency_does_not_match_aggregated_output() { - let l = 1; - let link = C2aToC4aShareCommitmentLink { l }; - - // C2: 1 party × 1 modulus = commit 99 - let c2 = c2_signals(&[make_field(99)]); - let source_values = link.extract_source_values(&c2); - - // C4: row 0 = [5], aggregated output = 99 - // commit 99 is only in the tail — must not match - let c4 = c4_signals(&[vec![make_field(5)]], make_field(99)); - - assert!(!link.check_consistency(&source_values, &c4, 0, 0)); - } - - #[test] - fn short_or_empty_signals() { - let link = C2aToC4aShareCommitmentLink { l: 2 }; - assert!(link.extract_source_values(&[0u8; 32]).is_empty()); - assert!(!link.check_consistency(&[], &[0u8; 256], 0, 0)); - assert!(!link.check_consistency(&[make_field(1)], &[0u8; 16], 0, 0)); - } - - #[test] - fn c2b_to_c4b_variant() { - let l = 2; - let link = C2bToC4bShareCommitmentLink { l }; - let c2 = c2_signals(&[make_field(7), make_field(8)]); - let source_values = link.extract_source_values(&c2); - - let c4 = c4_signals(&[vec![make_field(7), make_field(8)]], make_field(0)); - assert!(link.check_consistency(&source_values, &c4, 0, 0)); - - let c4_wrong = c4_signals(&[vec![make_field(7), make_field(9)]], make_field(0)); - assert!(!link.check_consistency(&source_values, &c4_wrong, 0, 0)); - } -} +mod tests; diff --git a/crates/zk-prover/src/commitment_links/c2_to_c4/tests.rs b/crates/zk-prover/src/commitment_links/c2_to_c4/tests.rs new file mode 100644 index 0000000000..6ce304165a --- /dev/null +++ b/crates/zk-prover/src/commitment_links/c2_to_c4/tests.rs @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +fn make_field(val: u8) -> [u8; 32] { + let mut f = [0u8; 32]; + f[31] = val; + f +} + +/// C2 inner signals: [expected_secret_commitment] + share commitments (row-major: party, mod). +fn c2_signals(share_commits: &[[u8; 32]]) -> Vec { + let mut v = Vec::new(); + v.extend_from_slice(&make_field(0xFF)); // expected_secret_commitment (skipped) + for c in share_commits { + v.extend_from_slice(c); + } + v +} + +/// C4 signals: [expected_commitments row-major (party, mod)..., aggregated_commitment]. +fn c4_signals(rows: &[Vec<[u8; 32]>], aggregated: [u8; 32]) -> Vec { + let mut v = Vec::new(); + for row in rows { + for c in row { + v.extend_from_slice(c); + } + } + v.extend_from_slice(&aggregated); + v +} + +#[test] +fn extract_share_commitments_from_c2() { + let link = C2aToC4aShareCommitmentLink { l: 2 }; + // C2 with 3 parties × 2 moduli = 6 share commits + let commits: Vec<[u8; 32]> = (1u8..=6).map(make_field).collect(); + let c2 = c2_signals(&commits); + let values = link.extract_source_values(&c2); + assert_eq!(values.len(), 6); + assert_eq!(values[0], make_field(1)); + assert_eq!(values[5], make_field(6)); +} + +#[test] +fn extract_skips_secret_commitment() { + let link = C2aToC4aShareCommitmentLink { l: 2 }; + let c2 = c2_signals(&[make_field(1), make_field(2)]); + let values = link.extract_source_values(&c2); + assert_eq!(values.len(), 2); + assert!(!values.contains(&make_field(0xFF))); +} + +/// 3 parties (N=3), 2 moduli (L=2), 2 honest parties (H=2). +/// C2 from party X=1, C4 for recipient R=1. +/// C2_X: [p0m0,p0m1, p1m0,p1m1, p2m0,p2m1] +/// C4_R=1: rows for party 0 and party 1 = [[p0m0,p0m1],[p1m0,p1m1]] + agg +/// check_consistency(src=1, tgt=1): C2 slot R=1 = [p1m0,p1m1] must equal C4 row X=1 = [p1m0,p1m1] +#[test] +fn consistency_passes_precise_l_way_check() { + let l = 2; + let link = C2aToC4aShareCommitmentLink { l }; + + // C2 from sender X=1: 3 parties × 2 moduli + let c2 = c2_signals(&[ + make_field(10), + make_field(11), // party 0 + make_field(20), + make_field(21), // party 1 (slot for tgt_party=1) + make_field(30), + make_field(31), // party 2 + ]); + let source_values = link.extract_source_values(&c2); + + // C4 for recipient R=1: 2 honest parties (rows for X=0 and X=1) + let c4 = c4_signals( + &[ + vec![make_field(10), make_field(11)], // row X=0 + vec![make_field(20), make_field(21)], // row X=1 (sender's commits for this recipient) + ], + make_field(99), // aggregated output + ); + + // src_party_id=1 (sender X=1), tgt_party_id=1 (recipient R=1) + assert!(link.check_consistency(&source_values, &c4, 1, 1)); +} + +#[test] +fn consistency_fails_when_wrong_modulus_commitment() { + let l = 2; + let link = C2aToC4aShareCommitmentLink { l }; + + let c2 = c2_signals(&[ + make_field(10), + make_field(11), + make_field(20), + make_field(21), // party 1 slot + make_field(30), + make_field(31), + ]); + let source_values = link.extract_source_values(&c2); + + // C4 has correct first modulus (20) but wrong second (99 instead of 21) + let c4 = c4_signals( + &[ + vec![make_field(10), make_field(11)], + vec![make_field(20), make_field(99)], // second modulus wrong + ], + make_field(0), + ); + + assert!(!link.check_consistency(&source_values, &c4, 1, 1)); +} + +#[test] +fn consistency_fails_when_wrong_party_slot() { + let l = 2; + let link = C2aToC4aShareCommitmentLink { l }; + + let c2 = c2_signals(&[ + make_field(10), + make_field(11), // party 0 + make_field(20), + make_field(21), // party 1 + ]); + let source_values = link.extract_source_values(&c2); + + // C4 has party-0 commits in row 0 only + let c4 = c4_signals(&[vec![make_field(10), make_field(11)]], make_field(0)); + + // src=0, tgt=1: C2 slot 1 = [20,21], C4 row 0 = [10,11] — mismatch + assert!(!link.check_consistency(&source_values, &c4, 0, 1)); +} + +#[test] +fn consistency_does_not_match_aggregated_output() { + let l = 1; + let link = C2aToC4aShareCommitmentLink { l }; + + // C2: 1 party × 1 modulus = commit 99 + let c2 = c2_signals(&[make_field(99)]); + let source_values = link.extract_source_values(&c2); + + // C4: row 0 = [5], aggregated output = 99 + // commit 99 is only in the tail — must not match + let c4 = c4_signals(&[vec![make_field(5)]], make_field(99)); + + assert!(!link.check_consistency(&source_values, &c4, 0, 0)); +} + +#[test] +fn short_or_empty_signals() { + let link = C2aToC4aShareCommitmentLink { l: 2 }; + assert!(link.extract_source_values(&[0u8; 32]).is_empty()); + assert!(!link.check_consistency(&[], &[0u8; 256], 0, 0)); + assert!(!link.check_consistency(&[make_field(1)], &[0u8; 16], 0, 0)); +} + +#[test] +fn c2b_to_c4b_variant() { + let l = 2; + let link = C2bToC4bShareCommitmentLink { l }; + let c2 = c2_signals(&[make_field(7), make_field(8)]); + let source_values = link.extract_source_values(&c2); + + let c4 = c4_signals(&[vec![make_field(7), make_field(8)]], make_field(0)); + assert!(link.check_consistency(&source_values, &c4, 0, 0)); + + let c4_wrong = c4_signals(&[vec![make_field(7), make_field(9)]], make_field(0)); + assert!(!link.check_consistency(&source_values, &c4_wrong, 0, 0)); +} diff --git a/crates/zk-prover/src/actors/commitment_links/c4a_to_c6.rs b/crates/zk-prover/src/commitment_links/c4a_to_c6.rs similarity index 100% rename from crates/zk-prover/src/actors/commitment_links/c4a_to_c6.rs rename to crates/zk-prover/src/commitment_links/c4a_to_c6.rs diff --git a/crates/zk-prover/src/actors/commitment_links/c4b_to_c6.rs b/crates/zk-prover/src/commitment_links/c4b_to_c6.rs similarity index 100% rename from crates/zk-prover/src/actors/commitment_links/c4b_to_c6.rs rename to crates/zk-prover/src/commitment_links/c4b_to_c6.rs diff --git a/crates/zk-prover/src/actors/commitment_links/c6_to_c7.rs b/crates/zk-prover/src/commitment_links/c6_to_c7.rs similarity index 100% rename from crates/zk-prover/src/actors/commitment_links/c6_to_c7.rs rename to crates/zk-prover/src/commitment_links/c6_to_c7.rs diff --git a/crates/zk-prover/src/actors/commitment_links/mod.rs b/crates/zk-prover/src/commitment_links/mod.rs similarity index 100% rename from crates/zk-prover/src/actors/commitment_links/mod.rs rename to crates/zk-prover/src/commitment_links/mod.rs diff --git a/crates/zk-prover/src/domain/mod.rs b/crates/zk-prover/src/domain.rs similarity index 68% rename from crates/zk-prover/src/domain/mod.rs rename to crates/zk-prover/src/domain.rs index d712663486..a9df8b64de 100644 --- a/crates/zk-prover/src/domain/mod.rs +++ b/crates/zk-prover/src/domain.rs @@ -4,13 +4,15 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -//! Pure, synchronous, unit-testable domain services for ZK proof orchestration. +//! Compatibility view of pure ZK protocol calculations and validation. //! //! These modules contain NO actix / `BusHandle` / `Addr` / signing concerns. //! The actors in [`crate::actors`] are thin transport shells that drive these //! state machines and perform all I/O (publishing, signing, persistence). +#[path = "commitment_links/mod.rs"] +pub(crate) mod commitment_links; +#[path = "node_proof_aggregation/workflow.rs"] pub(crate) mod node_dkg_fold; -pub(crate) mod proof_request; +#[path = "proof_verification/workflow.rs"] pub(crate) mod proof_verification; -pub(crate) mod share_verification; diff --git a/crates/zk-prover/src/domain/proof_request.rs b/crates/zk-prover/src/domain/proof_request.rs deleted file mode 100644 index f18be5894a..0000000000 --- a/crates/zk-prover/src/domain/proof_request.rs +++ /dev/null @@ -1,545 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Pure, synchronous domain logic for proof-request dispatch and completion. -//! -//! The [`crate::actors::proof_request::ProofRequestActor`] is a thin transport -//! shell: it owns the event bus and signer and performs all publish/sign I/O. -//! This module owns the business logic — the per-E3 pending-proof state machines -//! (which proofs have arrived, when a set is complete) and the deterministic -//! dispatch *planning* (which proof requests to emit, in what order, with which -//! `seq` index). It has NO actix / `BusHandle` / signing concerns. - -use std::collections::HashMap; -use std::sync::Arc; - -use e3_events::{ - DkgShareDecryptionProofRequest, E3id, EncryptionKey, EventContext, PkAggregationProofRequest, - PkGenerationProofRequest, Proof, Sequenced, ShareComputationProofRequest, - ShareEncryptionProofRequest, ThresholdShare, ZkRequest, -}; -use e3_utils::utility_types::ArcBytes; - -/// Identifies which threshold (C1/C2/C3) proof a response corresponds to. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum ThresholdProofKind { - PkGeneration, - SkShareComputation, - ESmShareComputation, - SkShareEncryption { - recipient_party_id: usize, - row_index: usize, - }, - ESmShareEncryption { - esi_index: usize, - recipient_party_id: usize, - row_index: usize, - }, -} - -/// Identifies which C4 (DkgShareDecryption) proof a response corresponds to. -#[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) enum DecryptionProofKind { - SecretKey, - SmudgingNoise { esi_idx: usize }, -} - -/// Per-E3 metadata for streaming DKG inner proof aggregation. -#[derive(Clone, Debug)] -pub(crate) struct NodeAggregationMeta { - pub(crate) party_id: u64, - pub(crate) total_expected: usize, - /// Buffered C0 proof, if it arrived before meta was stored. - pub(crate) pending_c0: Option, - /// When false, skip emitting DKGInnerProofReady (no recursive DKG aggregation). - pub(crate) proof_aggregation_enabled: bool, -} - -impl NodeAggregationMeta { - /// Total expected inner proofs for streaming aggregation: - /// C0..C4 (4 + sk + esm + 2). Mirrors the node-fold collector sizing. - pub(crate) fn total_expected_for(sk_enc_count: usize, e_sm_enc_count: usize) -> usize { - 4 + sk_enc_count + e_sm_enc_count + 2 - } - - /// Base `seq` for the first C4 proof: just after all C0..C3 proofs. - pub(crate) fn c4_base_seq(&self) -> usize { - self.total_expected.saturating_sub(2) - } -} - -#[derive(Clone, Debug)] -pub(crate) struct PendingProofRequest { - pub(crate) e3_id: E3id, - pub(crate) key: Arc, -} - -#[derive(Clone, Debug)] -pub(crate) struct PendingThresholdProofs { - pub(crate) e3_id: E3id, - pub(crate) full_share: Arc, - pub(crate) ec: EventContext, - pub(crate) pk_generation_proof: Option, - pub(crate) sk_share_computation_proof: Option, - pub(crate) e_sm_share_computation_proof: Option, - /// C3a proofs: keyed by (recipient_party_id, row_index) - pub(crate) sk_share_encryption_proofs: HashMap<(usize, usize), Proof>, - pub(crate) expected_sk_enc_count: usize, - /// C3b proofs: keyed by (esi_index, recipient_party_id, row_index) - pub(crate) e_sm_share_encryption_proofs: HashMap<(usize, usize, usize), Proof>, - pub(crate) expected_e_sm_enc_count: usize, - /// Maps positional index to real party_id (from ThresholdSharePending). - pub(crate) recipient_party_ids: Vec, -} - -impl PendingThresholdProofs { - pub(crate) fn new( - e3_id: E3id, - full_share: Arc, - ec: EventContext, - expected_sk_enc_count: usize, - expected_e_sm_enc_count: usize, - recipient_party_ids: Vec, - ) -> Self { - Self { - e3_id, - full_share, - ec, - pk_generation_proof: None, - sk_share_computation_proof: None, - e_sm_share_computation_proof: None, - sk_share_encryption_proofs: HashMap::new(), - expected_sk_enc_count, - e_sm_share_encryption_proofs: HashMap::new(), - expected_e_sm_enc_count, - recipient_party_ids, - } - } - - pub(crate) fn is_complete(&self) -> bool { - self.pk_generation_proof.is_some() - && self.sk_share_computation_proof.is_some() - && self.e_sm_share_computation_proof.is_some() - && self.sk_share_encryption_proofs.len() == self.expected_sk_enc_count - && self.e_sm_share_encryption_proofs.len() == self.expected_e_sm_enc_count - } - - pub(crate) fn store_proof(&mut self, kind: &ThresholdProofKind, proof: Proof) { - match kind { - ThresholdProofKind::PkGeneration => self.pk_generation_proof = Some(proof), - ThresholdProofKind::SkShareComputation => self.sk_share_computation_proof = Some(proof), - ThresholdProofKind::ESmShareComputation => { - self.e_sm_share_computation_proof = Some(proof) - } - ThresholdProofKind::SkShareEncryption { - recipient_party_id, - row_index, - } => { - self.sk_share_encryption_proofs - .insert((*recipient_party_id, *row_index), proof); - } - ThresholdProofKind::ESmShareEncryption { - esi_index, - recipient_party_id, - row_index, - } => { - self.e_sm_share_encryption_proofs - .insert((*esi_index, *recipient_party_id, *row_index), proof); - } - } - } - - pub(crate) fn total_expected(&self) -> usize { - 3 + self.expected_sk_enc_count + self.expected_e_sm_enc_count - } - - pub(crate) fn total_received(&self) -> usize { - let base = [ - self.pk_generation_proof.is_some(), - self.sk_share_computation_proof.is_some(), - self.e_sm_share_computation_proof.is_some(), - ] - .iter() - .filter(|&&v| v) - .count(); - base + self.sk_share_encryption_proofs.len() + self.e_sm_share_encryption_proofs.len() - } -} - -/// Pending C4 (DkgShareDecryption) proof generation state. -#[derive(Clone, Debug)] -pub(crate) struct PendingDecryptionProofs { - pub(crate) party_id: u64, - pub(crate) node: String, - pub(crate) ec: EventContext, - pub(crate) sk_proof: Option, - pub(crate) esm_proofs: HashMap, - pub(crate) expected_esm_count: usize, -} - -impl PendingDecryptionProofs { - pub(crate) fn is_complete(&self) -> bool { - self.sk_proof.is_some() - && self.esm_proofs.len() == self.expected_esm_count - && (0..self.expected_esm_count).all(|i| self.esm_proofs.contains_key(&i)) - } -} - -/// Pending C5 (PkAggregation) proof generation state. -#[derive(Clone, Debug)] -pub(crate) struct PendingPkAggregationProof { - pub(crate) ec: EventContext, - #[allow(dead_code)] - pub(crate) request: PkAggregationProofRequest, -} - -/// Pending C6 (ShareDecryptionProof) proof generation state. -#[derive(Clone, Debug)] -pub(crate) struct PendingShareDecryptionProof { - pub(crate) party_id: u64, - pub(crate) node: String, - pub(crate) decryption_share: Vec, - pub(crate) ec: EventContext, -} - -/// Pending C7 (DecryptedSharesAggregation) proof generation state. -#[derive(Clone, Debug)] -pub(crate) struct PendingAggregationProof { - pub(crate) ec: EventContext, -} - -/// A single planned threshold (C1/C2/C3) proof request: which proof, its `seq` -/// index for streaming aggregation, and the [`ZkRequest`] to dispatch. -pub(crate) struct ThresholdDispatchItem { - pub(crate) kind: ThresholdProofKind, - pub(crate) seq: usize, - pub(crate) request: ZkRequest, -} - -/// Build the deterministic, ordered set of C1/C2/C3 proof requests for a -/// `ThresholdSharePending` event. Pure: assigns the canonical `seq` indices -/// (C1=1, C2a=2, C2b=3, C3a[i]=4+i, C3b[j]=4+sk_count+j) and wraps each request. -pub(crate) fn plan_threshold_dispatch( - proof_request: PkGenerationProofRequest, - sk_share_computation_request: ShareComputationProofRequest, - e_sm_share_computation_request: ShareComputationProofRequest, - sk_share_encryption_requests: Vec, - e_sm_share_encryption_requests: Vec, -) -> Vec { - let sk_enc_count = sk_share_encryption_requests.len(); - let mut items = Vec::with_capacity(3 + sk_enc_count + e_sm_share_encryption_requests.len()); - - items.push(ThresholdDispatchItem { - kind: ThresholdProofKind::PkGeneration, - seq: 1, - request: ZkRequest::PkGeneration(proof_request), - }); - items.push(ThresholdDispatchItem { - kind: ThresholdProofKind::SkShareComputation, - seq: 2, - request: ZkRequest::ShareComputation(sk_share_computation_request), - }); - items.push(ThresholdDispatchItem { - kind: ThresholdProofKind::ESmShareComputation, - seq: 3, - request: ZkRequest::ShareComputation(e_sm_share_computation_request), - }); - - for (i, req) in sk_share_encryption_requests.into_iter().enumerate() { - let kind = ThresholdProofKind::SkShareEncryption { - recipient_party_id: req.recipient_party_id, - row_index: req.row_index, - }; - items.push(ThresholdDispatchItem { - kind, - seq: 4 + i, - request: ZkRequest::ShareEncryption(req), - }); - } - - for (j, req) in e_sm_share_encryption_requests.into_iter().enumerate() { - let kind = ThresholdProofKind::ESmShareEncryption { - esi_index: req.esi_index, - recipient_party_id: req.recipient_party_id, - row_index: req.row_index, - }; - items.push(ThresholdDispatchItem { - kind, - seq: 4 + sk_enc_count + j, - request: ZkRequest::ShareEncryption(req), - }); - } - - items -} - -/// A single planned C4 (DkgShareDecryption) proof request. -pub(crate) struct DecryptionDispatchItem { - pub(crate) kind: DecryptionProofKind, - pub(crate) seq: usize, - pub(crate) request: ZkRequest, -} - -/// Build the ordered set of C4 proof requests (SecretKey then SmudgingNoise[i]). -/// `c4_base_seq` is the streaming-aggregation `seq` of the C4a (SecretKey) proof; -/// each C4b proof follows at `c4_base_seq + 1 + esi_idx`. -pub(crate) fn plan_decryption_dispatch( - sk_request: DkgShareDecryptionProofRequest, - esm_requests: Vec, - c4_base_seq: usize, -) -> Vec { - let mut items = Vec::with_capacity(1 + esm_requests.len()); - items.push(DecryptionDispatchItem { - kind: DecryptionProofKind::SecretKey, - seq: c4_base_seq, - request: ZkRequest::DkgShareDecryption(sk_request), - }); - for (esi_idx, esm_req) in esm_requests.into_iter().enumerate() { - items.push(DecryptionDispatchItem { - kind: DecryptionProofKind::SmudgingNoise { esi_idx }, - seq: c4_base_seq + 1 + esi_idx, - request: ZkRequest::DkgShareDecryption(esm_req), - }); - } - items -} - -#[cfg(test)] -mod tests { - use super::*; - use e3_crypto::SensitiveBytes; - use e3_events::CircuitName; - use e3_fhe_params::BfvPreset; - use e3_trbfv::shares::BfvEncryptedShares; - use e3_zk_helpers::{computation::DkgInputType, CiphernodesCommitteeSize}; - - fn ec() -> EventContext { - use e3_events::{InterfoldEventData, TestEvent, Unsequenced}; - EventContext::::from(InterfoldEventData::from(TestEvent::new("x", 0))) - .sequence(0) - } - - fn sensitive() -> SensitiveBytes { - SensitiveBytes::from_encrypted(&[]) - } - - fn full_share() -> Arc { - Arc::new(ThresholdShare { - party_id: 0, - pk_share: ArcBytes::from_bytes(&[]), - sk_sss: BfvEncryptedShares::default(), - esi_sss: vec![], - }) - } - - fn share_computation_req() -> ShareComputationProofRequest { - ShareComputationProofRequest { - secret_raw: sensitive(), - secret_sss_raw: sensitive(), - dkg_input_type: DkgInputType::SecretKey, - params_preset: BfvPreset::default(), - committee_size: CiphernodesCommitteeSize::Micro, - } - } - - fn pk_generation_req() -> PkGenerationProofRequest { - PkGenerationProofRequest { - pk0_share: ArcBytes::from_bytes(&[]), - sk: sensitive(), - eek: sensitive(), - e_sm: sensitive(), - params_preset: BfvPreset::default(), - committee_size: CiphernodesCommitteeSize::Micro, - } - } - - fn share_encryption_req( - recipient_party_id: usize, - row_index: usize, - esi_index: usize, - ) -> ShareEncryptionProofRequest { - ShareEncryptionProofRequest { - share_row_raw: sensitive(), - ciphertext_raw: ArcBytes::from_bytes(&[]), - recipient_pk_raw: ArcBytes::from_bytes(&[]), - u_rns_raw: sensitive(), - e0_rns_raw: sensitive(), - e1_rns_raw: sensitive(), - dkg_input_type: DkgInputType::SecretKey, - params_preset: BfvPreset::default(), - committee_size: CiphernodesCommitteeSize::Micro, - recipient_party_id, - row_index, - esi_index, - } - } - - fn dkg_share_decryption_req() -> DkgShareDecryptionProofRequest { - DkgShareDecryptionProofRequest { - sk_bfv: sensitive(), - honest_ciphertexts_raw: vec![], - num_honest_parties: 0, - num_moduli: 0, - own_plaintext_idx: 0, - own_share_raw: sensitive(), - dkg_input_type: DkgInputType::SecretKey, - params_preset: BfvPreset::default(), - committee_size: CiphernodesCommitteeSize::Micro, - } - } - - fn proof(seed: u8) -> Proof { - Proof::new( - CircuitName::PkAggregation, - ArcBytes::from_bytes(&[seed]), - ArcBytes::from_bytes(&[seed.wrapping_add(1)]), - ) - } - - fn pending(sk: usize, esm: usize) -> PendingThresholdProofs { - PendingThresholdProofs::new( - E3id::new("1", 1), - full_share(), - ec(), - sk, - esm, - vec![1, 2, 3], - ) - } - - #[test] - fn threshold_completes_only_when_all_proofs_present() { - let mut p = pending(1, 1); - assert!(!p.is_complete()); - assert_eq!(p.total_expected(), 3 + 1 + 1); - assert_eq!(p.total_received(), 0); - - p.store_proof(&ThresholdProofKind::PkGeneration, proof(1)); - p.store_proof(&ThresholdProofKind::SkShareComputation, proof(2)); - p.store_proof(&ThresholdProofKind::ESmShareComputation, proof(3)); - assert!(!p.is_complete()); - assert_eq!(p.total_received(), 3); - - p.store_proof( - &ThresholdProofKind::SkShareEncryption { - recipient_party_id: 2, - row_index: 0, - }, - proof(4), - ); - assert!(!p.is_complete()); - p.store_proof( - &ThresholdProofKind::ESmShareEncryption { - esi_index: 0, - recipient_party_id: 2, - row_index: 0, - }, - proof(5), - ); - assert!(p.is_complete()); - assert_eq!(p.total_received(), 5); - } - - #[test] - fn store_proof_dedupes_by_key() { - let mut p = pending(2, 0); - let key = ThresholdProofKind::SkShareEncryption { - recipient_party_id: 2, - row_index: 0, - }; - p.store_proof(&key, proof(4)); - p.store_proof(&key, proof(9)); // same (recipient,row) overwrites - assert_eq!(p.sk_share_encryption_proofs.len(), 1); - assert!(!p.is_complete()); // still expecting 2 distinct sk enc proofs - } - - #[test] - fn decryption_completes_when_sk_and_all_esm_present() { - let mut d = PendingDecryptionProofs { - party_id: 7, - node: "n".into(), - ec: ec(), - sk_proof: None, - esm_proofs: HashMap::new(), - expected_esm_count: 2, - }; - assert!(!d.is_complete()); - d.sk_proof = Some(proof(1)); - d.esm_proofs.insert(0, proof(2)); - assert!(!d.is_complete()); - d.esm_proofs.insert(1, proof(3)); - assert!(d.is_complete()); - } - - #[test] - fn decryption_requires_contiguous_esm_indices() { - let mut d = PendingDecryptionProofs { - party_id: 7, - node: "n".into(), - ec: ec(), - sk_proof: Some(proof(1)), - esm_proofs: HashMap::new(), - expected_esm_count: 2, - }; - // Two entries but indices {0,2} — count matches but index 1 missing. - d.esm_proofs.insert(0, proof(2)); - d.esm_proofs.insert(2, proof(3)); - assert!(!d.is_complete()); - } - - #[test] - fn node_agg_meta_seq_helpers() { - assert_eq!(NodeAggregationMeta::total_expected_for(2, 1), 4 + 2 + 1 + 2); - let meta = NodeAggregationMeta { - party_id: 0, - total_expected: NodeAggregationMeta::total_expected_for(2, 1), - pending_c0: None, - proof_aggregation_enabled: true, - }; - // c4_base_seq sits just after C0..C3 = total_expected - 2. - assert_eq!(meta.c4_base_seq(), 4 + 2 + 1); - } - - #[test] - fn threshold_plan_assigns_canonical_seqs() { - let enc = - |recipient: usize, row: usize, esi: usize| share_encryption_req(recipient, row, esi); - let plan = plan_threshold_dispatch( - pk_generation_req(), - share_computation_req(), - share_computation_req(), - vec![enc(2, 0, 0), enc(3, 0, 0)], - vec![enc(2, 0, 0)], - ); - let seqs: Vec = plan.iter().map(|i| i.seq).collect(); - assert_eq!(seqs, vec![1, 2, 3, 4, 5, 6]); - assert!(matches!(plan[0].kind, ThresholdProofKind::PkGeneration)); - assert!(matches!( - plan[3].kind, - ThresholdProofKind::SkShareEncryption { .. } - )); - assert!(matches!( - plan[5].kind, - ThresholdProofKind::ESmShareEncryption { .. } - )); - } - - #[test] - fn decryption_plan_assigns_offset_seqs() { - let plan = plan_decryption_dispatch( - dkg_share_decryption_req(), - vec![dkg_share_decryption_req(), dkg_share_decryption_req()], - 7, - ); - let seqs: Vec = plan.iter().map(|i| i.seq).collect(); - assert_eq!(seqs, vec![7, 8, 9]); - assert!(matches!(plan[0].kind, DecryptionProofKind::SecretKey)); - assert!(matches!( - plan[1].kind, - DecryptionProofKind::SmudgingNoise { esi_idx: 0 } - )); - } -} diff --git a/crates/zk-prover/src/domain/share_verification.rs b/crates/zk-prover/src/domain/share_verification.rs deleted file mode 100644 index 9e3c8dc1eb..0000000000 --- a/crates/zk-prover/src/domain/share_verification.rs +++ /dev/null @@ -1,1213 +0,0 @@ -// SPDX-License-Identifier: LGPL-3.0-only -// -// This file is provided WITHOUT ANY WARRANTY; -// without even the implied warranty of MERCHANTABILITY -// or FITNESS FOR A PARTICULAR PURPOSE. - -//! Pure, synchronous domain logic for C1/C2/C3/C4/C6 share-proof verification. -//! -//! The [`crate::actors::share_verification::ShareVerificationActor`] is a thin -//! transport shell: it owns the event bus and performs all publish/persist I/O. -//! This module owns the business logic — ECDSA validation, proof-commitment -//! hashing, consistency filtering, and ZK-result tallying — as pure functions on -//! the stateless [`ShareVerifier`] service, plus the per-E3 pending-state types. -//! It has NO actix / `BusHandle` / `Addr` concerns (tracing is allowed). - -use std::collections::{BTreeSet, HashMap, HashSet}; - -use alloy::primitives::{keccak256, Address, Bytes}; -use alloy::sol_types::SolValue; -use e3_events::{ - E3id, EventContext, PartyProofData, PartyProofsToVerify, PartyShareDecryptionProofsToVerify, - PartyVerificationResult, ProofType, Sequenced, SignedProofPayload, VerificationKind, -}; -use e3_utils::utility_types::ArcBytes; -use e3_zk_helpers::CiphernodesCommitteeSize; -use tracing::{info, warn}; - -/// Trait for party types whose signed proofs can be ECDSA-validated and ZK-verified. -pub(crate) trait VerifiableParty: Clone + PartialEq { - fn party_id(&self) -> u64; - fn signed_proofs(&self) -> Vec; -} - -impl VerifiableParty for PartyProofsToVerify { - fn party_id(&self) -> u64 { - self.sender_party_id - } - fn signed_proofs(&self) -> Vec { - self.signed_proofs.clone() - } -} - -impl VerifiableParty for PartyShareDecryptionProofsToVerify { - fn party_id(&self) -> u64 { - self.sender_party_id - } - fn signed_proofs(&self) -> Vec { - std::iter::once(self.signed_sk_decryption_proof.clone()) - .chain(self.signed_e_sm_decryption_proofs.iter().cloned()) - .collect() - } -} - -/// ECDSA validation result for a single party. -pub(crate) struct EcdsaPartyResult { - pub(crate) passed: bool, - /// The pair (signed_payload, recovered_address) of the first failing proof, if any. - pub(crate) failed_payload: Option<(SignedProofPayload, Option
)>, -} - -/// A single ECDSA failure to be attributed (emitted) by the actor. -pub(crate) struct EcdsaFailure { - pub(crate) party_id: u64, - pub(crate) signed: SignedProofPayload, - pub(crate) recovered: Option
, -} - -/// Outcome of validating + preparing a batch of party proofs for the -/// consistency-check + ZK phases. Pure data; the actor performs the I/O. -pub(crate) struct EcdsaValidationOutcome

{ - pub(crate) ecdsa_dishonest: HashSet, - /// Failures to emit, in party iteration order. - pub(crate) failures: Vec, - pub(crate) ecdsa_passed_parties: Vec

, - pub(crate) party_addresses: HashMap, - pub(crate) party_proof_hashes: HashMap>, - pub(crate) party_public_signals: HashMap>, - pub(crate) party_proof_data: HashMap>, - /// Assembled per-party data for the consistency-check request. - pub(crate) consistency_party_data: Vec, -} - -/// Pending verification state — stored while ZK verification is in flight. -pub(crate) struct PendingVerification { - pub(crate) e3_id: E3id, - pub(crate) kind: VerificationKind, - pub(crate) ec: EventContext, - /// Parties that failed ECDSA (dishonest before ZK runs). - pub(crate) ecdsa_dishonest: HashSet, - /// Pre-dishonest parties from the dispatch (missing/incomplete proofs). - pub(crate) pre_dishonest: BTreeSet, - /// Party IDs dispatched for ZK verification (for cross-checking results). - pub(crate) dispatched_party_ids: HashSet, - /// Recovered address for each party (from ECDSA step). - pub(crate) party_addresses: HashMap, - /// Cached (proof_type, data_hash) per party — for emitting ProofVerificationPassed. - pub(crate) party_proof_hashes: HashMap>, - /// Cached (proof_type, public_signals) per party — for commitment consistency checking. - pub(crate) party_public_signals: HashMap>, - /// Parallel to `party_public_signals` — raw `proof.data` per (party, proof_type). - pub(crate) party_proof_data: HashMap>, - /// BFV preset for circuit artifact resolution. - #[allow(dead_code)] - pub(crate) params_preset: e3_fhe_params::BfvPreset, - /// Committee size for per-committee circuit artifact resolution. - #[allow(dead_code)] - pub(crate) committee_size: CiphernodesCommitteeSize, -} - -/// Pending consistency check — stored between ECDSA pass and ZK dispatch. -pub(crate) struct PendingConsistencyCheck { - pub(crate) e3_id: E3id, - pub(crate) kind: VerificationKind, - pub(crate) ec: EventContext, - /// Parties that failed ECDSA (dishonest before consistency runs). - pub(crate) ecdsa_dishonest: HashSet, - /// Pre-dishonest parties from the dispatch (missing/incomplete proofs). - pub(crate) pre_dishonest: BTreeSet, - /// Recovered address per ECDSA-passed party. - pub(crate) party_addresses: HashMap, - /// (proof_type, data_hash) per party — for ProofVerificationPassed after ZK. - pub(crate) party_proof_hashes: HashMap>, - /// (proof_type, public_signals) per party — for consistency & ZK. - pub(crate) party_public_signals: HashMap>, - /// Parallel to `party_public_signals` — raw `proof.data` per (party, proof_type). - pub(crate) party_proof_data: HashMap>, - /// Original ECDSA-passed share proofs for ZK dispatch. - pub(crate) ecdsa_passed_share_proofs: Vec, - /// Original ECDSA-passed decryption proofs for ZK dispatch. - pub(crate) ecdsa_passed_decryption_proofs: Vec, - /// BFV preset for circuit artifact resolution. - pub(crate) params_preset: e3_fhe_params::BfvPreset, - /// Committee size for per-committee circuit artifact resolution. - pub(crate) committee_size: CiphernodesCommitteeSize, -} - -/// Filter out inconsistent parties and collect dispatched party IDs. -/// Returns `None` if all parties were filtered out (nothing to verify). -pub(crate) fn filter_consistent

( - proofs: Vec

, - inconsistent: &BTreeSet, - party_id_of: impl Fn(&P) -> u64, -) -> Option<(Vec

, HashSet)> { - let passed: Vec

= proofs - .into_iter() - .filter(|p| !inconsistent.contains(&party_id_of(p))) - .collect(); - if passed.is_empty() { - return None; - } - let ids = passed.iter().map(party_id_of).collect(); - Some((passed, ids)) -} - -/// Per-party emission decision produced when tallying ZK verification results. -pub(crate) enum ZkPartyEmission { - /// Party failed ZK — attribute fault using the signed payload. - Failed { - party_id: u64, - signed: SignedProofPayload, - }, - /// Party passed ZK — emit `ProofVerificationPassed` for each cached proof. - Passed { party_id: u64 }, -} - -/// Outcome of tallying ZK verification results: the accumulated dishonest set -/// and the ordered emission decisions. -pub(crate) struct ZkTallyOutcome { - pub(crate) dishonest: BTreeSet, - pub(crate) emissions: Vec, -} - -/// Human-readable label for a verification kind (used in log lines). -pub(crate) fn label_for(kind: &VerificationKind) -> &'static str { - match kind { - VerificationKind::ShareProofs => "C2/C3", - VerificationKind::ThresholdDecryptionProofs => "C6", - VerificationKind::PkGenerationProofs => "C1", - VerificationKind::DecryptionProofs => "C4", - } -} - -/// Stateless service holding all pure share-verification business logic. -pub(crate) struct ShareVerifier; - -impl ShareVerifier { - /// Keccak256 over `abi_encode((proof.data, proof.public_signals))`. - fn proof_data_hash(signed: &SignedProofPayload) -> [u8; 32] { - let msg = ( - Bytes::copy_from_slice(&signed.payload.proof.data), - Bytes::copy_from_slice(&signed.payload.proof.public_signals), - ) - .abi_encode(); - keccak256(&msg).into() - } - - /// Check that a party supplied the canonical proof-type layout for this protocol phase. - /// - /// C2/C3 counts are derived from the threshold parameter preset. Variable C4b and C6 counts - /// are checked against trusted local state by their producers. This trust-boundary check - /// prevents a signed proof for another phase (or a duplicated singleton proof) from satisfying - /// the current phase merely because its self-declared [`ProofType`] maps to a valid circuit. - fn has_canonical_proof_shape( - kind: &VerificationKind, - signed_proofs: &[SignedProofPayload], - params_preset: e3_fhe_params::BfvPreset, - ) -> bool { - match kind { - VerificationKind::PkGenerationProofs => { - signed_proofs.len() == 1 - && signed_proofs[0].payload.proof_type == ProofType::C1PkGeneration - } - VerificationKind::ShareProofs => { - // Canonical order is C2a, C2b, C3a x L, C3b x L. Each C3 proof encrypts one - // modulus row of the threshold-parameter Shamir secret, even though encryption - // itself uses the paired DKG parameters. The dispatch currently carries that DKG - // preset, so recover its threshold counterpart before deriving L. - let threshold_preset = params_preset - .threshold_counterpart() - .unwrap_or(params_preset); - let num_share_rows = threshold_preset.metadata().num_moduli; - signed_proofs.len() == 2 + (2 * num_share_rows) - && signed_proofs[0].payload.proof_type == ProofType::C2aSkShareComputation - && signed_proofs[1].payload.proof_type == ProofType::C2bESmShareComputation - && signed_proofs[2..2 + num_share_rows] - .iter() - .all(|signed| signed.payload.proof_type == ProofType::C3aSkShareEncryption) - && signed_proofs[2 + num_share_rows..] - .iter() - .all(|signed| signed.payload.proof_type == ProofType::C3bESmShareEncryption) - } - VerificationKind::DecryptionProofs => { - // PartyShareDecryptionProofsToVerify has one distinguished C4a slot followed - // by one or more C4b slots. The producer checks the exact C4b count against - // `es_poly_sum`; here we bind every signed payload to its structural role because - // C4a/C4b share a CircuitName. - signed_proofs.len() >= 2 - && signed_proofs[0].payload.proof_type == ProofType::C4aSkShareDecryption - && signed_proofs[1..] - .iter() - .all(|signed| signed.payload.proof_type == ProofType::C4bESmShareDecryption) - } - VerificationKind::ThresholdDecryptionProofs => { - !signed_proofs.is_empty() - && signed_proofs.iter().all(|signed| { - signed.payload.proof_type == ProofType::C6ThresholdShareDecryption - }) - } - } - } - - /// Validate ECDSA properties for a set of signed proofs from one party: - /// 1. e3_id match - /// 2. Signature recovery (valid ECDSA) - /// 3. Recovered signer owns the canonical finalized-committee party slot - /// 4. Signer consistency (all proofs from same address) - /// 5. Circuit name matches expected ProofType circuits - pub(crate) fn ecdsa_validate_signed_proofs( - sender_party_id: u64, - signed_proofs: &[SignedProofPayload], - e3_id_str: &str, - label: &str, - expected_signer: Option

, - ) -> EcdsaPartyResult { - if signed_proofs.is_empty() { - info!( - "{} party {} supplied an empty signed-proof bundle", - label, sender_party_id - ); - return EcdsaPartyResult { - passed: false, - failed_payload: None, - }; - } - - let Some(expected_signer) = expected_signer else { - info!( - "{} party {} has no canonical finalized-committee slot", - label, sender_party_id - ); - return EcdsaPartyResult { - passed: false, - // The outer party id is not part of the signed payload. Its absence from the - // canonical committee is therefore a structural dispatch failure, not - // self-authenticating evidence that can safely be attributed to the signer. - failed_payload: None, - }; - }; - - let mut expected_addr: Option
= None; - - for signed in signed_proofs { - // 1. e3_id match - if signed.payload.e3_id.to_string() != e3_id_str { - info!( - "{} proof from party {} has wrong e3_id ({} vs {})", - label, sender_party_id, signed.payload.e3_id, e3_id_str - ); - return EcdsaPartyResult { - passed: false, - failed_payload: Some((signed.clone(), expected_addr)), - }; - } - - // 2. Signature recovery - match signed.recover_address() { - Ok(addr) => { - // 3. Canonical party ownership and signer consistency - if addr != expected_signer { - info!( - "{} proof signer {} does not own party {} (expected {})", - label, addr, sender_party_id, expected_signer - ); - return EcdsaPartyResult { - passed: false, - failed_payload: Some((signed.clone(), Some(addr))), - }; - } - match &expected_addr { - Some(ea) if *ea != addr => { - info!( - "{} inconsistent signer for party {}", - label, sender_party_id - ); - return EcdsaPartyResult { - passed: false, - failed_payload: Some((signed.clone(), Some(addr))), - }; - } - None => expected_addr = Some(addr), - _ => {} - } - } - Err(e) => { - info!( - "{} signature recovery failed for party {} ({:?}): {}", - label, sender_party_id, signed.payload.proof_type, e - ); - return EcdsaPartyResult { - passed: false, - failed_payload: Some((signed.clone(), expected_addr)), - }; - } - } - - // 4. Circuit name validation - let expected_circuits = signed.payload.proof_type.circuit_names(); - if !expected_circuits.contains(&signed.payload.proof.circuit) { - info!( - "{} circuit mismatch for party {}: expected {:?}, got {:?}", - label, sender_party_id, expected_circuits, signed.payload.proof.circuit - ); - return EcdsaPartyResult { - passed: false, - failed_payload: Some((signed.clone(), expected_addr)), - }; - } - } - - EcdsaPartyResult { - passed: true, - failed_payload: None, - } - } - - /// Run ECDSA validation across all parties and prepare the cached proof - /// hashes/signals/data plus the consistency-check request payload for the - /// parties that passed. Pure: no event publishing. - pub(crate) fn validate_and_prepare( - party_proofs: &[P], - e3_id_str: &str, - kind: &VerificationKind, - label: &str, - committee: Option<&[Address]>, - params_preset: e3_fhe_params::BfvPreset, - committee_size: CiphernodesCommitteeSize, - ) -> EcdsaValidationOutcome

{ - let mut ecdsa_dishonest = HashSet::new(); - let mut failures = Vec::new(); - let mut ecdsa_passed_parties = Vec::new(); - let mut party_addresses: HashMap = HashMap::new(); - - // A finalized committee is a one-to-one party-slot map. The on-chain sortition path - // guarantees unique operators, but replayed/test-produced events also cross this - // boundary. Refuse an ambiguous map rather than allowing one signer to own two slots. - let expected_committee_n = committee_size.values().n; - let committee = committee.filter(|members| { - let unique_members: HashSet

= members.iter().copied().collect(); - let is_unique = unique_members.len() == members.len(); - let has_expected_size = members.len() == expected_committee_n; - if !is_unique { - warn!( - "{} finalized committee contains duplicate addresses; rejecting proof batch", - label - ); - } - if !has_expected_size { - warn!( - "{} finalized committee has {} members, expected {}; rejecting proof batch", - label, - members.len(), - expected_committee_n - ); - } - is_unique && has_expected_size - }); - - // Collapse byte-identical replayed bundles idempotently. Two different bundles for the - // same outer party id are ambiguous and are both excluded, so a canonical party can - // contribute at most one verification result and one set of passed-proof emissions. - let mut unique_parties: Vec<&P> = Vec::new(); - let mut party_positions: HashMap = HashMap::new(); - let mut conflicting_party_ids = HashSet::new(); - for party in party_proofs { - match party_positions.get(&party.party_id()).copied() { - Some(position) if unique_parties[position] == party => { - warn!( - "{} duplicate replay for party {} ignored", - label, - party.party_id() - ); - } - Some(_) => { - warn!( - "{} conflicting proof bundles for party {}; rejecting that party", - label, - party.party_id() - ); - conflicting_party_ids.insert(party.party_id()); - } - None => { - party_positions.insert(party.party_id(), unique_parties.len()); - unique_parties.push(party); - } - } - } - - for party in unique_parties { - if conflicting_party_ids.contains(&party.party_id()) { - ecdsa_dishonest.insert(party.party_id()); - continue; - } - - let proofs = party.signed_proofs(); - if !Self::has_canonical_proof_shape(kind, &proofs, params_preset) { - info!( - "{} party {} supplied a non-canonical proof-type multiplicity/order", - label, - party.party_id() - ); - // The verification kind and outer bundle are not signed. Exclude the party, - // but do not manufacture slash evidence from an otherwise valid signed proof. - ecdsa_dishonest.insert(party.party_id()); - continue; - } - - let expected_signer = usize::try_from(party.party_id()) - .ok() - .and_then(|party_id| committee.and_then(|members| members.get(party_id))) - .copied(); - let result = Self::ecdsa_validate_signed_proofs( - party.party_id(), - &proofs, - e3_id_str, - label, - expected_signer, - ); - if result.passed { - ecdsa_passed_parties.push(party.clone()); - } else { - ecdsa_dishonest.insert(party.party_id()); - if let Some((signed, addr)) = result.failed_payload { - failures.push(EcdsaFailure { - party_id: party.party_id(), - signed, - recovered: addr, - }); - } - } - } - - // Store recovered addresses for passed parties. - for party in &ecdsa_passed_parties { - let proofs = party.signed_proofs(); - if let Some(first_signed) = proofs.first() { - if let Ok(addr) = first_signed.recover_address() { - party_addresses.insert(party.party_id(), addr); - } - } - } - - // Compute proof hashes and public signals for ECDSA-passed parties. - let mut party_proof_hashes: HashMap> = HashMap::new(); - let mut party_public_signals: HashMap> = HashMap::new(); - let mut party_raw_proof_data: HashMap> = HashMap::new(); - for party in &ecdsa_passed_parties { - let hashes: Vec<(ProofType, [u8; 32])> = party - .signed_proofs() - .iter() - .map(|signed| (signed.payload.proof_type, Self::proof_data_hash(signed))) - .collect(); - let signals: Vec<(ProofType, ArcBytes)> = party - .signed_proofs() - .iter() - .map(|signed| { - ( - signed.payload.proof_type, - signed.payload.proof.public_signals.clone(), - ) - }) - .collect(); - let datas: Vec<(ProofType, ArcBytes)> = party - .signed_proofs() - .iter() - .map(|signed| (signed.payload.proof_type, signed.payload.proof.data.clone())) - .collect(); - party_proof_hashes.insert(party.party_id(), hashes); - party_public_signals.insert(party.party_id(), signals); - party_raw_proof_data.insert(party.party_id(), datas); - } - - // Assemble consistency-check request payload. - let consistency_party_data: Vec = ecdsa_passed_parties - .iter() - .map(|party| { - let signals = party_public_signals - .get(&party.party_id()) - .cloned() - .unwrap_or_default(); - let hashes = party_proof_hashes - .get(&party.party_id()) - .cloned() - .unwrap_or_default(); - let raw_datas = party_raw_proof_data - .get(&party.party_id()) - .cloned() - .unwrap_or_default(); - let proofs = signals - .into_iter() - .zip(hashes) - .zip(raw_datas) - .map(|(((pt, ps), (_, dh)), (_, pd))| (pt, ps, dh, pd)) - .collect(); - PartyProofData { - party_id: party.party_id(), - address: party_addresses - .get(&party.party_id()) - .copied() - .unwrap_or_default(), - proofs, - } - }) - .collect(); - - EcdsaValidationOutcome { - ecdsa_dishonest, - failures, - ecdsa_passed_parties, - party_addresses, - party_proof_hashes, - party_public_signals, - party_proof_data: party_raw_proof_data, - consistency_party_data, - } - } - - /// Tally ZK verification results against the dispatched set: accumulate the - /// dishonest party set (including parties missing from the response) and - /// produce ordered per-party emission decisions. Pure: no event publishing. - pub(crate) fn tally_zk_results( - pre_dishonest: BTreeSet, - ecdsa_dishonest: &HashSet, - dispatched_party_ids: &HashSet, - zk_results: &[PartyVerificationResult], - ) -> ZkTallyOutcome { - let mut dishonest: BTreeSet = pre_dishonest; - dishonest.extend(ecdsa_dishonest); - - // Canonicalize the worker response before tallying. Exact duplicate responses are - // harmless replays and are collapsed; conflicting responses for one party fail closed. - // This also guarantees at most one Passed/Failed emission per canonical party. - let mut unique_results: HashMap = HashMap::new(); - let mut result_order = Vec::new(); - let mut conflicting_party_ids = HashSet::new(); - for result in zk_results { - if !dispatched_party_ids.contains(&result.sender_party_id) { - warn!( - "ZK result for party {} was not dispatched — ignoring", - result.sender_party_id - ); - continue; - } - - match unique_results.get(&result.sender_party_id).copied() { - Some(existing) if existing == result => { - warn!( - "Duplicate ZK result replay for party {} ignored", - result.sender_party_id - ); - } - Some(_) => { - warn!( - "Conflicting ZK results for party {} — treating as dishonest", - result.sender_party_id - ); - conflicting_party_ids.insert(result.sender_party_id); - } - None => { - unique_results.insert(result.sender_party_id, result); - result_order.push(result.sender_party_id); - } - } - } - - // Cross-check: every dispatched party must appear exactly once after replay collapse. - for &dispatched_pid in dispatched_party_ids { - if !unique_results.contains_key(&dispatched_pid) { - warn!( - "Party {} was dispatched for ZK verification but missing from results — treating as dishonest", - dispatched_pid - ); - dishonest.insert(dispatched_pid); - } - } - - let mut emissions = Vec::new(); - for party_id in result_order { - if conflicting_party_ids.contains(&party_id) { - dishonest.insert(party_id); - continue; - } - let Some(result) = unique_results.get(&party_id) else { - // `result_order` and `unique_results` are populated together above. Keep this - // boundary panic-free if that implementation changes later. - dishonest.insert(party_id); - continue; - }; - if !result.all_verified { - dishonest.insert(result.sender_party_id); - if let Some(ref signed) = result.failed_signed_payload { - emissions.push(ZkPartyEmission::Failed { - party_id: result.sender_party_id, - signed: signed.clone(), - }); - } - } else { - emissions.push(ZkPartyEmission::Passed { - party_id: result.sender_party_id, - }); - } - } - - ZkTallyOutcome { - dishonest, - emissions, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use alloy::signers::local::PrivateKeySigner; - use e3_events::{Proof, ProofPayload, ProofType}; - use e3_fhe_params::BfvPreset; - - fn signer() -> PrivateKeySigner { - PrivateKeySigner::random() - } - - fn minimum_committee(mut members: Vec
) -> Vec
{ - while members.len() < CiphernodesCommitteeSize::Minimum.values().n { - let candidate = signer().address(); - if !members.contains(&candidate) { - members.push(candidate); - } - } - members - } - - fn signed_proof( - s: &PrivateKeySigner, - e3_id: &E3id, - proof_type: ProofType, - marker: u8, - ) -> SignedProofPayload { - let proof = Proof::new( - proof_type.circuit_names()[0], - ArcBytes::from_bytes(&[marker, 2, 3]), - ArcBytes::from_bytes(&[4, 5, marker]), - ); - let payload = ProofPayload { - e3_id: e3_id.clone(), - proof_type, - proof, - }; - SignedProofPayload::sign(payload, s).expect("sign") - } - - fn signed_share_bundle( - s: &PrivateKeySigner, - e3_id: &E3id, - num_share_rows: usize, - ) -> Vec { - let mut proofs = vec![ - signed_proof(s, e3_id, ProofType::C2aSkShareComputation, 2), - signed_proof(s, e3_id, ProofType::C2bESmShareComputation, 3), - ]; - for _ in 0..num_share_rows { - proofs.push(signed_proof(s, e3_id, ProofType::C3aSkShareEncryption, 4)); - } - for _ in 0..num_share_rows { - proofs.push(signed_proof(s, e3_id, ProofType::C3bESmShareEncryption, 5)); - } - proofs - } - - /// Build a signed C1 (PkGeneration) proof for `party_id` under `e3_id`, - /// optionally with a deliberately wrong circuit name. - fn signed_pk(s: &PrivateKeySigner, e3_id: &E3id, wrong_circuit: bool) -> SignedProofPayload { - use e3_events::CircuitName; - let proof_type = ProofType::C1PkGeneration; - if !wrong_circuit { - return signed_proof(s, e3_id, proof_type, 1); - } - let circuit = if wrong_circuit { - CircuitName::PkBfv - } else { - proof_type.circuit_names()[0] - }; - let proof = Proof::new( - circuit, - ArcBytes::from_bytes(&[1, 2, 3]), - ArcBytes::from_bytes(&[4, 5, 6]), - ); - let payload = ProofPayload { - e3_id: e3_id.clone(), - proof_type, - proof, - }; - SignedProofPayload::sign(payload, s).expect("sign") - } - - fn e3() -> E3id { - E3id::new("1", 1) - } - - #[test] - fn ecdsa_passes_for_well_formed_proof() { - let s = signer(); - let e3 = e3(); - let p = signed_pk(&s, &e3, false); - let res = ShareVerifier::ecdsa_validate_signed_proofs( - 7, - &[p], - &e3.to_string(), - "C1", - Some(s.address()), - ); - assert!(res.passed); - assert!(res.failed_payload.is_none()); - } - - #[test] - fn ecdsa_fails_on_wrong_e3_id() { - let s = signer(); - let p = signed_pk(&s, &e3(), false); - let res = - ShareVerifier::ecdsa_validate_signed_proofs(7, &[p], "999/0", "C1", Some(s.address())); - assert!(!res.passed); - assert!(res.failed_payload.is_some()); - } - - #[test] - fn ecdsa_fails_on_circuit_mismatch() { - let s = signer(); - let e3 = e3(); - let p = signed_pk(&s, &e3, true); - let res = ShareVerifier::ecdsa_validate_signed_proofs( - 7, - &[p], - &e3.to_string(), - "C1", - Some(s.address()), - ); - assert!(!res.passed); - } - - #[test] - fn ecdsa_fails_on_inconsistent_signer() { - let s1 = signer(); - let s2 = signer(); - let e3 = e3(); - let p1 = signed_pk(&s1, &e3, false); - let p2 = signed_pk(&s2, &e3, false); - let res = ShareVerifier::ecdsa_validate_signed_proofs( - 7, - &[p1, p2], - &e3.to_string(), - "C1", - Some(s1.address()), - ); - assert!(!res.passed); - } - - #[test] - fn ecdsa_fails_when_signer_does_not_own_party_slot() { - let proof_signer = signer(); - let slot_owner = signer(); - let e3 = e3(); - let proof = signed_pk(&proof_signer, &e3, false); - - let result = ShareVerifier::ecdsa_validate_signed_proofs( - 1, - &[proof], - &e3.to_string(), - "C1", - Some(slot_owner.address()), - ); - - assert!(!result.passed); - let (_, recovered) = result.failed_payload.expect("attributable mismatch"); - assert_eq!(recovered, Some(proof_signer.address())); - } - - #[test] - fn ecdsa_fails_for_empty_bundle_or_missing_party_slot() { - let e3 = e3(); - let owner = signer(); - let empty = ShareVerifier::ecdsa_validate_signed_proofs( - 0, - &[], - &e3.to_string(), - "C1", - Some(owner.address()), - ); - assert!(!empty.passed); - assert!(empty.failed_payload.is_none()); - - let proof = signed_pk(&owner, &e3, false); - let missing = - ShareVerifier::ecdsa_validate_signed_proofs(3, &[proof], &e3.to_string(), "C1", None); - assert!(!missing.passed); - } - - #[test] - fn prepare_rejects_one_signer_relabelled_across_other_party_slots() { - let first = signer(); - let second = signer(); - let third = signer(); - let e3 = e3(); - let parties = vec![ - PartyProofsToVerify { - sender_party_id: 0, - signed_proofs: vec![signed_pk(&first, &e3, false)], - }, - PartyProofsToVerify { - sender_party_id: 1, - // A valid proof from party 0 cannot fill party 1's slot. - signed_proofs: vec![signed_pk(&first, &e3, false)], - }, - PartyProofsToVerify { - sender_party_id: 2, - // Nor can the same signer fill any other canonical slot. - signed_proofs: vec![signed_pk(&first, &e3, false)], - }, - ]; - let committee = [first.address(), second.address(), third.address()]; - - let outcome = ShareVerifier::validate_and_prepare( - &parties, - &e3.to_string(), - &VerificationKind::PkGenerationProofs, - "C1", - Some(&committee), - BfvPreset::InsecureDkg512, - CiphernodesCommitteeSize::Minimum, - ); - - assert_eq!(outcome.ecdsa_passed_parties.len(), 1); - assert_eq!(outcome.ecdsa_passed_parties[0].sender_party_id, 0); - assert_eq!(outcome.ecdsa_dishonest, HashSet::from([1, 2])); - } - - #[test] - fn canonical_shape_rejects_cross_phase_and_singleton_multiplicity() { - let s = signer(); - let e3 = e3(); - - let c1 = signed_proof(&s, &e3, ProofType::C1PkGeneration, 1); - assert!(ShareVerifier::has_canonical_proof_shape( - &VerificationKind::PkGenerationProofs, - std::slice::from_ref(&c1), - BfvPreset::InsecureDkg512, - )); - assert!(!ShareVerifier::has_canonical_proof_shape( - &VerificationKind::PkGenerationProofs, - &[c1.clone(), c1.clone()], - BfvPreset::InsecureDkg512, - )); - assert!(!ShareVerifier::has_canonical_proof_shape( - &VerificationKind::ThresholdDecryptionProofs, - std::slice::from_ref(&c1), - BfvPreset::InsecureDkg512, - )); - let c6 = signed_proof(&s, &e3, ProofType::C6ThresholdShareDecryption, 9); - assert!(ShareVerifier::has_canonical_proof_shape( - &VerificationKind::ThresholdDecryptionProofs, - std::slice::from_ref(&c6), - BfvPreset::InsecureDkg512, - )); - assert!(!ShareVerifier::has_canonical_proof_shape( - &VerificationKind::ThresholdDecryptionProofs, - &[], - BfvPreset::InsecureDkg512, - )); - - let share_bundle = signed_share_bundle(&s, &e3, 2); - assert!(ShareVerifier::has_canonical_proof_shape( - &VerificationKind::ShareProofs, - &share_bundle, - BfvPreset::InsecureDkg512, - )); - let mut duplicate_c2a = share_bundle.clone(); - duplicate_c2a.insert(1, duplicate_c2a[0].clone()); - assert!(!ShareVerifier::has_canonical_proof_shape( - &VerificationKind::ShareProofs, - &duplicate_c2a, - BfvPreset::InsecureDkg512, - )); - - let secure_share_bundle = signed_share_bundle(&s, &e3, 3); - assert!(ShareVerifier::has_canonical_proof_shape( - &VerificationKind::ShareProofs, - &secure_share_bundle, - BfvPreset::SecureThreshold8192, - )); - assert!(!ShareVerifier::has_canonical_proof_shape( - &VerificationKind::ShareProofs, - &share_bundle, - BfvPreset::SecureThreshold8192, - )); - - let c4_bundle = vec![ - signed_proof(&s, &e3, ProofType::C4aSkShareDecryption, 6), - signed_proof(&s, &e3, ProofType::C4bESmShareDecryption, 7), - ]; - assert!(ShareVerifier::has_canonical_proof_shape( - &VerificationKind::DecryptionProofs, - &c4_bundle, - BfvPreset::InsecureDkg512, - )); - assert!(!ShareVerifier::has_canonical_proof_shape( - &VerificationKind::DecryptionProofs, - &c4_bundle[1..], - BfvPreset::InsecureDkg512, - )); - let mut extra_c4b = c4_bundle.clone(); - extra_c4b.push(c4_bundle[1].clone()); - assert!(ShareVerifier::has_canonical_proof_shape( - &VerificationKind::DecryptionProofs, - &extra_c4b, - BfvPreset::InsecureDkg512, - )); - let mut wrong_c4_tail = c4_bundle.clone(); - wrong_c4_tail.push(c6); - assert!(!ShareVerifier::has_canonical_proof_shape( - &VerificationKind::DecryptionProofs, - &wrong_c4_tail, - BfvPreset::InsecureDkg512, - )); - } - - #[test] - fn share_shape_uses_threshold_secret_rows_when_dispatch_carries_dkg_preset() { - let s = signer(); - let e3 = e3(); - - // Production dispatch carries the share-encryption (DKG) preset, but C3 requests are - // generated from rows of the paired threshold-parameter Shamir secret. - let insecure_production_bundle = signed_share_bundle(&s, &e3, 2); - assert_eq!(BfvPreset::InsecureDkg512.metadata().num_moduli, 1); - assert_eq!(BfvPreset::InsecureThreshold512.metadata().num_moduli, 2); - assert!(ShareVerifier::has_canonical_proof_shape( - &VerificationKind::ShareProofs, - &insecure_production_bundle, - BfvPreset::InsecureDkg512, - )); - assert!(!ShareVerifier::has_canonical_proof_shape( - &VerificationKind::ShareProofs, - &signed_share_bundle(&s, &e3, 1), - BfvPreset::InsecureDkg512, - )); - - let secure_production_bundle = signed_share_bundle(&s, &e3, 3); - assert_eq!(BfvPreset::SecureDkg8192.metadata().num_moduli, 2); - assert_eq!(BfvPreset::SecureThreshold8192.metadata().num_moduli, 3); - assert!(ShareVerifier::has_canonical_proof_shape( - &VerificationKind::ShareProofs, - &secure_production_bundle, - BfvPreset::SecureDkg8192, - )); - assert!(!ShareVerifier::has_canonical_proof_shape( - &VerificationKind::ShareProofs, - &signed_share_bundle(&s, &e3, 2), - BfvPreset::SecureDkg8192, - )); - } - - #[test] - fn prepare_excludes_wrong_phase_without_creating_slash_evidence() { - let s = signer(); - let e3 = e3(); - let parties = [PartyProofsToVerify { - sender_party_id: 0, - signed_proofs: vec![signed_proof( - &s, - &e3, - ProofType::C6ThresholdShareDecryption, - 9, - )], - }]; - let committee = minimum_committee(vec![s.address()]); - - let outcome = ShareVerifier::validate_and_prepare( - &parties, - &e3.to_string(), - &VerificationKind::PkGenerationProofs, - "C1", - Some(&committee), - BfvPreset::InsecureDkg512, - CiphernodesCommitteeSize::Minimum, - ); - - assert!(outcome.ecdsa_passed_parties.is_empty()); - assert_eq!(outcome.ecdsa_dishonest, HashSet::from([0])); - assert!(outcome.failures.is_empty()); - } - - #[test] - fn prepare_collapses_identical_party_replay_and_rejects_conflict() { - let s = signer(); - let e3 = e3(); - let committee = minimum_committee(vec![s.address()]); - let party = PartyProofsToVerify { - sender_party_id: 0, - signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 1)], - }; - - let replayed = ShareVerifier::validate_and_prepare( - &[party.clone(), party.clone()], - &e3.to_string(), - &VerificationKind::PkGenerationProofs, - "C1", - Some(&committee), - BfvPreset::InsecureDkg512, - CiphernodesCommitteeSize::Minimum, - ); - assert_eq!(replayed.ecdsa_passed_parties, vec![party.clone()]); - assert!(replayed.ecdsa_dishonest.is_empty()); - assert_eq!(replayed.consistency_party_data.len(), 1); - - let conflicting = PartyProofsToVerify { - sender_party_id: 0, - signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 2)], - }; - let conflict = ShareVerifier::validate_and_prepare( - &[party, conflicting], - &e3.to_string(), - &VerificationKind::PkGenerationProofs, - "C1", - Some(&committee), - BfvPreset::InsecureDkg512, - CiphernodesCommitteeSize::Minimum, - ); - assert!(conflict.ecdsa_passed_parties.is_empty()); - assert_eq!(conflict.ecdsa_dishonest, HashSet::from([0])); - assert!(conflict.failures.is_empty()); - } - - #[test] - fn prepare_rejects_ambiguous_committee_where_one_signer_owns_multiple_slots() { - let s = signer(); - let e3 = e3(); - let parties = vec![ - PartyProofsToVerify { - sender_party_id: 0, - signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 1)], - }, - PartyProofsToVerify { - sender_party_id: 1, - signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 2)], - }, - ]; - let ambiguous_committee = [s.address(), s.address(), signer().address()]; - - let outcome = ShareVerifier::validate_and_prepare( - &parties, - &e3.to_string(), - &VerificationKind::PkGenerationProofs, - "C1", - Some(&ambiguous_committee), - BfvPreset::InsecureDkg512, - CiphernodesCommitteeSize::Minimum, - ); - - assert!(outcome.ecdsa_passed_parties.is_empty()); - assert_eq!(outcome.ecdsa_dishonest, HashSet::from([0, 1])); - assert!(outcome.failures.is_empty()); - } - - #[test] - fn prepare_rejects_committee_with_wrong_circuit_dimension() { - let s = signer(); - let e3 = e3(); - let parties = [PartyProofsToVerify { - sender_party_id: 0, - signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 1)], - }]; - let undersized_committee = [s.address()]; - - let outcome = ShareVerifier::validate_and_prepare( - &parties, - &e3.to_string(), - &VerificationKind::PkGenerationProofs, - "C1", - Some(&undersized_committee), - BfvPreset::InsecureDkg512, - CiphernodesCommitteeSize::Minimum, - ); - - assert!(outcome.ecdsa_passed_parties.is_empty()); - assert_eq!(outcome.ecdsa_dishonest, HashSet::from([0])); - assert!(outcome.failures.is_empty()); - } - - #[test] - fn filter_consistent_drops_inconsistent_and_returns_ids() { - let proofs = vec![1u64, 2, 3]; - let inconsistent: BTreeSet = [2].into_iter().collect(); - let (passed, ids) = filter_consistent(proofs, &inconsistent, |p| *p).expect("some"); - assert_eq!(passed, vec![1, 3]); - assert!(ids.contains(&1) && ids.contains(&3) && !ids.contains(&2)); - } - - #[test] - fn filter_consistent_returns_none_when_all_filtered() { - let proofs = vec![1u64, 2]; - let inconsistent: BTreeSet = [1, 2].into_iter().collect(); - assert!(filter_consistent(proofs, &inconsistent, |p| *p).is_none()); - } - - #[test] - fn tally_marks_missing_dispatched_party_dishonest() { - let dispatched: HashSet = [1, 2].into_iter().collect(); - let ecdsa: HashSet = HashSet::new(); - // No ZK results at all → both dispatched parties are missing → dishonest. - let out = ShareVerifier::tally_zk_results(BTreeSet::new(), &ecdsa, &dispatched, &[]); - assert!(out.dishonest.contains(&1)); - assert!(out.dishonest.contains(&2)); - assert!(out.emissions.is_empty()); - } - - #[test] - fn tally_collapses_identical_result_replay() { - let dispatched = HashSet::from([1]); - let result = PartyVerificationResult { - sender_party_id: 1, - all_verified: true, - failed_signed_payload: None, - recovered_address: None, - }; - - let out = ShareVerifier::tally_zk_results( - BTreeSet::new(), - &HashSet::new(), - &dispatched, - &[result.clone(), result], - ); - - assert!(out.dishonest.is_empty()); - assert_eq!(out.emissions.len(), 1); - assert!(matches!( - out.emissions[0], - ZkPartyEmission::Passed { party_id: 1 } - )); - } - - #[test] - fn tally_rejects_conflicting_results_for_one_party() { - let dispatched = HashSet::from([1]); - let passed = PartyVerificationResult { - sender_party_id: 1, - all_verified: true, - failed_signed_payload: None, - recovered_address: None, - }; - let failed = PartyVerificationResult { - all_verified: false, - ..passed.clone() - }; - - let out = ShareVerifier::tally_zk_results( - BTreeSet::new(), - &HashSet::new(), - &dispatched, - &[passed, failed], - ); - - assert_eq!(out.dishonest, BTreeSet::from([1])); - assert!(out.emissions.is_empty()); - } -} diff --git a/crates/zk-prover/src/lib.rs b/crates/zk-prover/src/lib.rs index 08948a970d..47fa5b765d 100644 --- a/crates/zk-prover/src/lib.rs +++ b/crates/zk-prover/src/lib.rs @@ -4,6 +4,7 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. +mod actor_system; mod actors; mod backend; mod circuits; @@ -16,13 +17,14 @@ mod prover; pub mod test_utils; mod traits; mod witness; +mod workflow; -pub use actors::commitment_links::default_links; +pub use actor_system::{setup_zk_actors, ZkActorRecovery, ZkActors}; pub use actors::{ - setup_zk_actors, CommitmentConsistencyCheckerExtension, ProofRequestActor, - ProofVerificationActor, ShareVerificationActor, ZkActorRecovery, ZkActors, - ZkVerificationRequest, ZkVerificationResponse, + CommitmentConsistencyCheckerExtension, ProofRequestActor, ProofVerificationActor, + ShareVerificationActor, ZkVerificationRequest, ZkVerificationResponse, }; +pub use domain::commitment_links::default_links; pub use backend::{SetupStatus, ZkBackend}; pub use circuits::aggregation::c3_accumulator::generate_sequential_c3_fold; diff --git a/crates/zk-prover/src/node_proof_aggregation/actor.rs b/crates/zk-prover/src/node_proof_aggregation/actor.rs new file mode 100644 index 0000000000..f8d857e482 --- /dev/null +++ b/crates/zk-prover/src/node_proof_aggregation/actor.rs @@ -0,0 +1,299 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Node-level DKG proof aggregation: buffer all inner proofs (C0–C4), then run one +//! [`ZkRequest::NodeDkgFold`] when [`ThresholdSharePending`] says the full set is ready. + +use std::collections::{BTreeMap, HashMap}; + +use actix::{Actor, Addr, Context, Handler}; +use alloy::primitives::Address; +use alloy::signers::local::PrivateKeySigner; +use e3_events::{ + BusHandle, ComputeRequest, ComputeRequestError, ComputeResponse, ComputeResponseKind, + CorrelationId, DKGInnerProofReady, DKGRecursiveAggregationComplete, DkgFoldAttestationPayload, + E3Failed, E3Stage, E3id, EventContext, EventPublisher, EventSubscriber, EventType, + FailureReason, InterfoldEvent, InterfoldEventData, Proof, Sequenced, SignedDkgFoldAttestation, + ThresholdSharePending, TypedEvent, ZkRequest, ZkResponse, +}; +use e3_fhe_params::build_pair_for_preset; +use tracing::{error, info, warn}; + +use crate::domain::node_dkg_fold::{DkgProofCollectionState, NodeDkgFoldMeta}; +use crate::node_fold_public::extract_node_fold_agg_commits; + +/// Actor that collects DKG inner proofs and dispatches a single [`ZkRequest::NodeDkgFold`]. +pub struct NodeProofAggregator { + bus: BusHandle, + signer: PrivateKeySigner, + /// Per-chain `DkgFoldAttestationVerifier` address (EIP-712 `verifyingContract`). + /// Looked up by `e3_id.chain_id()` when signing fold attestations. + dkg_fold_attestation_verifiers_by_chain: HashMap>, + states: HashMap, + fold_correlation: HashMap, + pending_inner_proofs: HashMap>, +} + +impl NodeProofAggregator { + pub fn new( + bus: &BusHandle, + signer: PrivateKeySigner, + dkg_fold_attestation_verifiers_by_chain: HashMap>, + ) -> Self { + Self { + bus: bus.clone(), + signer, + dkg_fold_attestation_verifiers_by_chain, + states: HashMap::new(), + fold_correlation: HashMap::new(), + pending_inner_proofs: HashMap::new(), + } + } + + pub fn setup( + bus: &BusHandle, + signer: PrivateKeySigner, + dkg_fold_attestation_verifiers_by_chain: HashMap>, + ) -> Addr { + let addr = Self::new(bus, signer, dkg_fold_attestation_verifiers_by_chain).start(); + bus.subscribe(EventType::ThresholdSharePending, addr.clone().into()); + bus.subscribe(EventType::DKGInnerProofReady, addr.clone().into()); + bus.subscribe(EventType::ComputeResponse, addr.clone().into()); + bus.subscribe(EventType::ComputeRequestError, addr.clone().into()); + addr + } +} + +#[path = "effects.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +#[cfg(test)] +mod tests { + use super::*; + use anyhow::Result; + use e3_events::{ + CircuitName, ComputeRequestErrorKind, ComputeRequestKind, Event, HistoryCollector, + NodeDkgFoldRequest, TakeEvents, Unsequenced, ZkError, + }; + use e3_test_helpers::get_common_setup; + use e3_zk_helpers::CiphernodesCommitteeSize; + + fn test_ctx(data: impl Into) -> EventContext { + EventContext::::from(data.into()).sequence(0) + } + + fn test_signer() -> PrivateKeySigner { + "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" + .parse() + .expect("test signer") + } + + fn dummy_proof(seed: u8) -> Proof { + Proof::new( + CircuitName::PkAggregation, + e3_utils::ArcBytes::from_bytes(&[seed]), + e3_utils::ArcBytes::from_bytes(&[seed.wrapping_add(1)]), + ) + } + + async fn next_event( + history: &Addr>, + ) -> Result { + let mut result = history.send(TakeEvents::::new(1)).await?; + assert!(!result.timed_out, "timed out waiting for an event"); + Ok(result.events.pop().expect("expected one event")) + } + + #[actix::test] + async fn node_dkg_fold_compute_error_emits_e3_failed() -> Result<()> { + let (bus, _rng, _seed, _params, _crp, _errors, history) = get_common_setup(None)?; + let mut aggregator = NodeProofAggregator::new(&bus, test_signer(), HashMap::new()); + let e3_id = E3id::new("42", 1); + let correlation_id = CorrelationId::new(); + + aggregator.states.insert( + e3_id.clone(), + DkgProofCollectionState { + meta: NodeDkgFoldMeta { + party_id: 7, + total_expected: 0, + sk_enc_count: 0, + e_sm_enc_count: 0, + sk_share_encryption_requests: Vec::new(), + e_sm_share_encryption_requests: Vec::new(), + committee_n: 0, + committee_h: 0, + n_moduli: 0, + params_preset: e3_fhe_params::BfvPreset::InsecureThreshold512, + committee_size: CiphernodesCommitteeSize::Minimum, + }, + buffer: BTreeMap::new(), + fold_correlation: Some(correlation_id), + last_ec: test_ctx(DKGRecursiveAggregationComplete { + e3_id: e3_id.clone(), + party_id: 7, + aggregated_proof: None, + fold_attestation: None, + }), + }, + ); + aggregator + .fold_correlation + .insert(correlation_id, e3_id.clone()); + + let request = ComputeRequest::zk( + ZkRequest::NodeDkgFold(NodeDkgFoldRequest { + c0_proof: dummy_proof(1), + c1_proof: dummy_proof(2), + c2a_proof: dummy_proof(3), + c2b_proof: dummy_proof(4), + c3a_inner_proofs: Vec::new(), + c3b_inner_proofs: Vec::new(), + c4a_proof: dummy_proof(5), + c4b_proof: dummy_proof(6), + c3_slot_indices_a: Vec::new(), + c3_slot_indices_b: Vec::new(), + c3_total_slots: 0, + party_id: 7, + params_preset: e3_fhe_params::BfvPreset::InsecureThreshold512, + committee_size: CiphernodesCommitteeSize::Minimum, + }), + correlation_id, + e3_id.clone(), + ); + + aggregator.handle_compute_request_error(TypedEvent::new( + ComputeRequestError::new( + ComputeRequestErrorKind::Zk(ZkError::ProofGenerationFailed("boom".to_string())), + request, + ), + test_ctx(DKGRecursiveAggregationComplete { + e3_id: e3_id.clone(), + party_id: 7, + aggregated_proof: None, + fold_attestation: None, + }), + )); + + let event = next_event(&history).await?; + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == e3_id + && data.failed_at_stage == E3Stage::CommitteeFinalized + && data.reason == FailureReason::DKGInvalidShares + )); + assert!(!aggregator.states.contains_key(&e3_id)); + assert!(aggregator.fold_correlation.is_empty()); + + Ok(()) + } + + #[actix::test] + async fn early_inner_proof_is_prebuffered_until_collection_starts() -> Result<()> { + let (bus, _rng, _seed, _params, _crp, _errors, history) = get_common_setup(None)?; + let mut aggregator = NodeProofAggregator::new(&bus, test_signer(), HashMap::new()); + let e3_id = E3id::new("43", 1); + let early_proof = dummy_proof(10); + + aggregator.handle_inner_proof_ready(TypedEvent::new( + DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: 7, + proof: early_proof.clone(), + seq: 0, + }, + test_ctx(DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: 7, + proof: early_proof.clone(), + seq: 0, + }), + )); + + assert_eq!( + aggregator + .pending_inner_proofs + .get(&e3_id) + .map(BTreeMap::len), + Some(1) + ); + + aggregator.initialize_collection_state( + e3_id.clone(), + NodeDkgFoldMeta { + party_id: 7, + total_expected: 6, + sk_enc_count: 0, + e_sm_enc_count: 0, + sk_share_encryption_requests: Vec::new(), + e_sm_share_encryption_requests: Vec::new(), + committee_n: 0, + committee_h: 0, + n_moduli: 0, + params_preset: e3_fhe_params::BfvPreset::InsecureThreshold512, + committee_size: CiphernodesCommitteeSize::Minimum, + }, + test_ctx(DKGRecursiveAggregationComplete { + e3_id: e3_id.clone(), + party_id: 7, + aggregated_proof: None, + fold_attestation: None, + }), + ); + + assert!(!aggregator.pending_inner_proofs.contains_key(&e3_id)); + assert_eq!( + aggregator + .states + .get(&e3_id) + .map(|state| state.buffer.len()), + Some(1) + ); + + for seq in 1..6 { + let proof = dummy_proof((10 + seq) as u8); + aggregator.handle_inner_proof_ready(TypedEvent::new( + DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: 7, + proof: proof.clone(), + seq, + }, + test_ctx(DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: 7, + proof, + seq, + }), + )); + } + + let event = next_event(&history).await?; + match event.into_data() { + InterfoldEventData::ComputeRequest(request) => { + assert_eq!(request.e3_id, e3_id); + match request.request { + ComputeRequestKind::Zk(ZkRequest::NodeDkgFold(fold_request)) => { + assert_eq!(fold_request.c0_proof, early_proof); + } + other => panic!("expected NodeDkgFold request, got {other:?}"), + } + } + other => panic!("expected ComputeRequest event, got {other:?}"), + } + + assert!(aggregator + .states + .get(&e3_id) + .and_then(|state| state.fold_correlation) + .is_some()); + + Ok(()) + } +} diff --git a/crates/zk-prover/src/node_proof_aggregation/effects.rs b/crates/zk-prover/src/node_proof_aggregation/effects.rs new file mode 100644 index 0000000000..60a991cffc --- /dev/null +++ b/crates/zk-prover/src/node_proof_aggregation/effects.rs @@ -0,0 +1,309 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! DKG inner-proof collection and node-fold dispatch. + +use super::*; + +impl NodeProofAggregator { + pub(in crate::actors::node_proof_aggregator) fn handle_threshold_share_pending( + &mut self, + msg: TypedEvent, + ) { + let (msg, ec) = msg.into_components(); + let e3_id = msg.e3_id.clone(); + + if !msg.proof_aggregation_enabled { + self.pending_inner_proofs.remove(&e3_id); + info!( + "NodeProofAggregator: proof aggregation disabled for E3 {} — skipping", + e3_id + ); + if let Err(err) = self.bus.publish( + DKGRecursiveAggregationComplete { + e3_id: e3_id.clone(), + party_id: msg.full_share.party_id, + aggregated_proof: None, + fold_attestation: None, + }, + ec, + ) { + error!( + "NodeProofAggregator: failed to publish DKGRecursiveAggregationComplete (skipped) for E3 {}: {err}", + e3_id + ); + } + return; + } + + let sk_enc_count = msg.sk_share_encryption_requests.len(); + let e_sm_enc_count = msg.e_sm_share_encryption_requests.len(); + let total_expected = NodeDkgFoldMeta::total_expected_for(sk_enc_count, e_sm_enc_count); + + let committee = msg.proof_request.committee_size.values(); + let (committee_n, committee_h, n_moduli) = + match build_pair_for_preset(msg.proof_request.params_preset) { + Ok((threshold_params, _)) => { + (committee.n, committee.h, threshold_params.moduli().len()) + } + Err(e) => { + self.pending_inner_proofs.remove(&e3_id); + error!( + "NodeProofAggregator: build_pair_for_preset failed for E3 {}: {e}", + e3_id + ); + let _ = self.bus.publish( + E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGInvalidShares, + }, + ec.clone(), + ); + return; + } + }; + + let meta = NodeDkgFoldMeta { + party_id: msg.full_share.party_id, + total_expected, + sk_enc_count, + e_sm_enc_count, + sk_share_encryption_requests: msg.sk_share_encryption_requests.clone(), + e_sm_share_encryption_requests: msg.e_sm_share_encryption_requests.clone(), + committee_n, + committee_h, + n_moduli, + params_preset: msg.proof_request.params_preset, + committee_size: msg.proof_request.committee_size, + }; + + info!( + "NodeProofAggregator: E3 {} party {} — expecting {} inner proofs (C0..C4) for NodeDkgFold", + e3_id, meta.party_id, total_expected, + ); + + self.initialize_collection_state(e3_id, meta, ec); + } + + pub(in crate::actors::node_proof_aggregator) fn handle_inner_proof_ready( + &mut self, + msg: TypedEvent, + ) { + let (msg, ec) = msg.into_components(); + let e3_id = msg.e3_id.clone(); + + let Some(state) = self.states.get_mut(&e3_id) else { + let pending = self.pending_inner_proofs.entry(e3_id.clone()).or_default(); + pending.insert(msg.seq, msg.proof); + warn!( + "NodeProofAggregator: received DKGInnerProofReady for E3 {} before ThresholdSharePending — prebuffered seq={} (have {})", + e3_id, + msg.seq, + pending.len() + ); + return; + }; + + if state.fold_correlation.is_some() { + warn!( + "NodeProofAggregator: seq={} arrived while NodeDkgFold in flight — dropped", + msg.seq + ); + return; + } + + state.buffer.insert(msg.seq, msg.proof); + state.last_ec = ec; + + info!( + "NodeProofAggregator: buffered seq={} for E3 {} (have {}/{})", + msg.seq, + e3_id, + state.buffer.len(), + state.meta.total_expected + ); + + self.try_dispatch_node_dkg_fold(&e3_id); + } + + pub(in crate::actors::node_proof_aggregator) fn initialize_collection_state( + &mut self, + e3_id: E3id, + meta: NodeDkgFoldMeta, + ec: EventContext, + ) { + let mut buffer = self.pending_inner_proofs.remove(&e3_id).unwrap_or_default(); + if !buffer.is_empty() { + info!( + "NodeProofAggregator: recovered {} prebuffered inner proofs for E3 {}", + buffer.len(), + e3_id + ); + } + + self.states.insert( + e3_id.clone(), + DkgProofCollectionState::new(meta, std::mem::take(&mut buffer), ec), + ); + + self.try_dispatch_node_dkg_fold(&e3_id); + } + + fn try_dispatch_node_dkg_fold(&mut self, e3_id: &E3id) { + let state = match self.states.get_mut(e3_id) { + Some(s) => s, + None => return, + }; + if !state.is_ready() { + return; + } + + let req = state.build_fold_request(); + let corr = CorrelationId::new(); + let ec = state.last_ec.clone(); + let party_id = state.meta.party_id; + + state.fold_correlation = Some(corr); + self.fold_correlation.insert(corr, e3_id.clone()); + + info!( + "NodeProofAggregator: dispatching NodeDkgFold for E3 {} party {}", + e3_id, party_id + ); + + if let Err(err) = self.bus.publish( + ComputeRequest::zk(ZkRequest::NodeDkgFold(req), corr, e3_id.clone()), + ec, + ) { + error!( + "NodeProofAggregator: failed to publish NodeDkgFold for E3 {}: {err}", + e3_id + ); + let _ = self.states.get_mut(e3_id).map(|s| { + s.fold_correlation = None; + }); + self.fold_correlation.remove(&corr); + } + } + + pub(in crate::actors::node_proof_aggregator) fn handle_node_dkg_response( + &mut self, + correlation_id: &CorrelationId, + proof: Proof, + ) { + let Some(e3_id) = self.fold_correlation.remove(correlation_id) else { + return; + }; + + let Some(state) = self.states.remove(&e3_id) else { + error!( + "NodeProofAggregator: NodeDkgFold response for unknown E3 {}", + e3_id + ); + return; + }; + + let party_id = state.meta.party_id; + let committee_n = state.meta.committee_n; + let committee_h = state.meta.committee_h; + let n_moduli = state.meta.n_moduli; + + let fold_attestation = match extract_node_fold_agg_commits( + &proof, + committee_n, + committee_h, + n_moduli, + ) { + Ok((extracted_party, commits)) => { + if extracted_party != party_id { + error!( + e3_id = %e3_id, + expected_party_id = party_id, + extracted_party_id = extracted_party, + "NodeFold public party_id does not match sortition party_id" + ); + None + } else if let Some(verifying_contract) = + self.dkg_fold_attestation_verifier_for(&e3_id) + { + let payload = DkgFoldAttestationPayload { + e3_id: e3_id.clone(), + verifying_contract, + party_id, + agg_commits: commits, + }; + match SignedDkgFoldAttestation::sign(payload, &self.signer) { + Ok(signed) => Some(signed), + Err(e) => { + error!( + e3_id = %e3_id, + party_id, + error = %e, + "failed to sign DkgFoldAttestation" + ); + None + } + } + } else { + error!( + e3_id = %e3_id, + party_id, + "NodeProofAggregator: cannot sign DkgFoldAttestation — CiphernodeRegistry.dkgFoldAttestationVerifier not configured" + ); + None + } + } + Err(e) => { + error!( + e3_id = %e3_id, + party_id, + error = %e, + "failed to extract sk_agg/esm_agg from NodeFold proof" + ); + None + } + }; + + if fold_attestation.is_none() { + error!( + e3_id = %e3_id, + party_id, + "NodeDkgFold succeeded but fold attestation missing — failing E3" + ); + if let Err(err) = self.bus.publish( + E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGInvalidShares, + }, + state.last_ec, + ) { + error!( + "NodeProofAggregator: failed to publish E3Failed for E3 {}: {err}", + e3_id + ); + } + return; + } + + info!( + "NodeProofAggregator: NodeDkgFold complete for E3 {} party {} — publishing DKGRecursiveAggregationComplete", + e3_id, party_id + ); + + if let Err(err) = self.bus.publish( + DKGRecursiveAggregationComplete { + e3_id: e3_id.clone(), + party_id, + aggregated_proof: Some(proof), + fold_attestation, + }, + state.last_ec, + ) { + error!( + "NodeProofAggregator: failed to publish DKGRecursiveAggregationComplete for E3 {}: {err}", + e3_id + ); + } + } +} diff --git a/crates/zk-prover/src/node_proof_aggregation/handlers.rs b/crates/zk-prover/src/node_proof_aggregation/handlers.rs new file mode 100644 index 0000000000..bd938d0304 --- /dev/null +++ b/crates/zk-prover/src/node_proof_aggregation/handlers.rs @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Event routing and compute-result handling. + +use super::*; + +impl Actor for NodeProofAggregator { + type Context = Context; +} + +impl Handler for NodeProofAggregator { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, _ctx: &mut Self::Context) -> Self::Result { + let (data, ec) = msg.into_components(); + match data { + InterfoldEventData::ThresholdSharePending(data) => { + self.handle_threshold_share_pending(TypedEvent::new(data, ec)); + } + InterfoldEventData::DKGInnerProofReady(data) => { + self.handle_inner_proof_ready(TypedEvent::new(data, ec)); + } + InterfoldEventData::ComputeResponse(data) => { + self.handle_compute_response(TypedEvent::new(data, ec)); + } + InterfoldEventData::ComputeRequestError(data) => { + self.handle_compute_request_error(TypedEvent::new(data, ec)); + } + _ => {} + } + } +} + +impl Handler> for NodeProofAggregator { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_threshold_share_pending(msg); + } +} + +impl Handler> for NodeProofAggregator { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_inner_proof_ready(msg); + } +} + +impl Handler> for NodeProofAggregator { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_compute_response(msg); + } +} + +impl Handler> for NodeProofAggregator { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_compute_request_error(msg); + } +} + +impl NodeProofAggregator { + pub(super) fn dkg_fold_attestation_verifier_for(&self, e3_id: &E3id) -> Option
{ + let chain_id = e3_id.chain_id(); + match self.dkg_fold_attestation_verifiers_by_chain.get(&chain_id) { + Some(Some(addr)) => Some(*addr), + Some(None) => None, + None => { + warn!( + chain_id, + "no dkgFoldAttestationVerifier configured for chain" + ); + None + } + } + } + + pub(super) fn handle_compute_response(&mut self, msg: TypedEvent) { + let (msg, _ec) = msg.into_components(); + if let ComputeResponseKind::Zk(ZkResponse::NodeDkgFold(resp)) = msg.response { + self.handle_node_dkg_response(&msg.correlation_id, resp.proof); + } + } + + pub(super) fn handle_compute_request_error(&mut self, msg: TypedEvent) { + let (msg, ec) = msg.into_components(); + if let Some(e3_id) = self.fold_correlation.remove(msg.correlation_id()) { + error!( + "NodeProofAggregator: NodeDkgFold failed for E3 {}: {:?} — aggregation aborted", + e3_id, + msg.get_err() + ); + let state = self.states.remove(&e3_id); + warn!( + "NodeProofAggregator: E3 {} NodeDkgFold failed — publishing E3Failed", + e3_id + ); + + if let Some(_state) = state { + if let Err(err) = self.bus.publish( + E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGInvalidShares, + }, + ec, + ) { + error!( + "NodeProofAggregator: failed to publish E3Failed for E3 {}: {err}", + e3_id + ); + } + } + } + } +} diff --git a/crates/zk-prover/src/domain/node_dkg_fold.rs b/crates/zk-prover/src/node_proof_aggregation/workflow.rs similarity index 100% rename from crates/zk-prover/src/domain/node_dkg_fold.rs rename to crates/zk-prover/src/node_proof_aggregation/workflow.rs diff --git a/crates/zk-prover/src/proof_request/actor.rs b/crates/zk-prover/src/proof_request/actor.rs new file mode 100644 index 0000000000..6a72f69468 --- /dev/null +++ b/crates/zk-prover/src/proof_request/actor.rs @@ -0,0 +1,106 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +use actix::{Actor, Addr, Context, Handler}; +use alloy::primitives::{keccak256, Bytes}; +use alloy::signers::local::PrivateKeySigner; +use alloy::sol_types::SolValue; +use e3_events::{ + AggregationProofPending, AggregationProofSigned, BusHandle, ComputeRequest, + ComputeRequestError, ComputeRequestErrorKind, ComputeResponse, ComputeResponseKind, + CorrelationId, DKGInnerProofReady, DecryptionKeyShared, DecryptionShareProofSigned, + DecryptionShareProofsPending, DecryptionshareCreated, DkgProofSigned, E3Failed, E3Stage, E3id, + EncryptionKeyCreated, EncryptionKeyPending, EventContext, EventPublisher, EventSubscriber, + EventType, FailureReason, InterfoldEvent, InterfoldEventData, PkAggregationProofPending, + PkAggregationProofSigned, PkBfvProofRequest, PkGenerationProofSigned, Proof, ProofPayload, + ProofType, ProofVerificationPassed, Sequenced, ShareDecryptionProofPending, SignedProofPayload, + ThresholdShareCreated, ThresholdSharePending, TypedEvent, ZkRequest, ZkResponse, +}; +use e3_utils::NotifySync; +use tracing::{error, info, trace, warn}; + +use crate::workflow::proof_request::{ + plan_decryption_dispatch, plan_threshold_dispatch, DecryptionProofKind, NodeAggregationMeta, + PendingAggregationProof, PendingDecryptionProofs, PendingPkAggregationProof, + PendingProofRequest, PendingShareDecryptionProof, PendingThresholdProofs, ThresholdProofKind, +}; + +/// Core actor that handles encryption key proof requests. +/// +/// Proofs are always wrapped in a [`SignedProofPayload`] before being published, +/// enabling fault attribution via the signed proof model. +/// A signer is required — if signing fails, the proof is not published. +pub struct ProofRequestActor { + bus: BusHandle, + signer: PrivateKeySigner, + pending: HashMap, + threshold_correlation: HashMap, + pending_threshold: HashMap, + /// C4 proof staging: correlation -> (e3_id, kind, seq) + decryption_correlation: HashMap, + /// Per-E3 metadata for DKGInnerProofReady emission. + node_agg_meta: HashMap, + /// C4 pending proofs per E3 + pending_decryption: HashMap, + /// C6 proof staging: correlation -> e3_id + share_decryption_correlation: HashMap, + /// C6 pending proofs per E3 + pending_share_decryption: HashMap, + /// C5 proof staging: correlation -> e3_id + pk_aggregation_correlation: HashMap, + /// C5 pending proofs per E3 + pending_pk_aggregation: HashMap, + /// C7 proof staging: correlation -> e3_id + aggregation_correlation: HashMap, + /// C7 pending proofs per E3 + pending_aggregation: HashMap, +} + +impl ProofRequestActor { + pub fn new(bus: &BusHandle, signer: PrivateKeySigner) -> Self { + Self { + bus: bus.clone(), + signer, + pending: HashMap::new(), + pending_threshold: HashMap::new(), + threshold_correlation: HashMap::new(), + decryption_correlation: HashMap::new(), + pending_decryption: HashMap::new(), + node_agg_meta: HashMap::new(), + share_decryption_correlation: HashMap::new(), + pending_share_decryption: HashMap::new(), + pk_aggregation_correlation: HashMap::new(), + pending_pk_aggregation: HashMap::new(), + aggregation_correlation: HashMap::new(), + pending_aggregation: HashMap::new(), + } + } + + pub fn setup(bus: &BusHandle, signer: PrivateKeySigner) -> Addr { + let addr = Self::new(bus, signer).start(); + bus.subscribe(EventType::EncryptionKeyPending, addr.clone().into()); + bus.subscribe(EventType::ComputeResponse, addr.clone().into()); + bus.subscribe(EventType::ComputeRequestError, addr.clone().into()); + bus.subscribe(EventType::ThresholdSharePending, addr.clone().into()); + bus.subscribe(EventType::DecryptionShareProofsPending, addr.clone().into()); + bus.subscribe(EventType::ShareDecryptionProofPending, addr.clone().into()); + bus.subscribe(EventType::PkAggregationProofPending, addr.clone().into()); + bus.subscribe(EventType::AggregationProofPending, addr.clone().into()); + addr + } +} + +#[path = "effects/mod.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +#[cfg(test)] +#[path = "actor_tests.rs"] +mod tests; diff --git a/crates/zk-prover/src/proof_request/actor_tests.rs b/crates/zk-prover/src/proof_request/actor_tests.rs new file mode 100644 index 0000000000..c9865f889a --- /dev/null +++ b/crates/zk-prover/src/proof_request/actor_tests.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use alloy::signers::local::PrivateKeySigner; +use anyhow::Result; +use e3_events::{ + ComputeRequestErrorKind, EncryptionKey, Event, HistoryCollector, TakeEvents, Unsequenced, + ZkError, +}; +use e3_test_helpers::get_common_setup; +use e3_utils::utility_types::ArcBytes; + +fn test_ctx(data: impl Into) -> EventContext { + EventContext::::from(data.into()).sequence(0) +} + +async fn next_event(history: &Addr>) -> Result { + let mut result = history.send(TakeEvents::::new(1)).await?; + assert!(!result.timed_out, "timed out waiting for an event"); + Ok(result.events.pop().expect("expected one event")) +} + +#[actix::test] +async fn c0_compute_error_emits_e3_failed() -> Result<()> { + let (bus, _rng, _seed, _params, _crp, _errors, history) = get_common_setup(None)?; + let mut actor = ProofRequestActor::new(&bus, PrivateKeySigner::random()); + let e3_id = E3id::new("44", 1); + let correlation_id = CorrelationId::new(); + + actor.pending.insert( + correlation_id, + PendingProofRequest { + e3_id: e3_id.clone(), + key: Arc::new(EncryptionKey::new(7, ArcBytes::from_bytes(&[1]))), + }, + ); + + actor.handle_compute_request_error(TypedEvent::new( + ComputeRequestError::new( + ComputeRequestErrorKind::Zk(ZkError::ProofGenerationFailed("boom".to_string())), + ComputeRequest::zk( + ZkRequest::PkBfv(PkBfvProofRequest::new( + ArcBytes::from_bytes(&[1]), + e3_fhe_params::BfvPreset::InsecureThreshold512, + e3_zk_helpers::CiphernodesCommitteeSize::Minimum, + )), + correlation_id, + e3_id.clone(), + ), + ), + test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGInvalidShares, + }), + )); + + let event = next_event(&history).await?; + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == e3_id + && data.failed_at_stage == E3Stage::CommitteeFinalized + && data.reason == FailureReason::DKGInvalidShares + )); + assert!(actor.pending.is_empty()); + + Ok(()) +} + +#[actix::test] +async fn decryption_failure_helper_emits_e3_failed() -> Result<()> { + let (bus, _rng, _seed, _params, _crp, _errors, history) = get_common_setup(None)?; + let actor = ProofRequestActor::new(&bus, PrivateKeySigner::random()); + let e3_id = E3id::new("45", 1); + + actor.fail_decryption_round( + e3_id.clone(), + &test_ctx(E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CiphertextReady, + reason: FailureReason::DecryptionInvalidShares, + }), + "test decryption failure", + ); + + let event = next_event(&history).await?; + assert!(matches!( + event.into_data(), + InterfoldEventData::E3Failed(data) + if data.e3_id == e3_id + && data.failed_at_stage == E3Stage::CiphertextReady + && data.reason == FailureReason::DecryptionInvalidShares + )); + + Ok(()) +} diff --git a/crates/zk-prover/src/proof_request/effects/aggregation_proofs.rs b/crates/zk-prover/src/proof_request/effects/aggregation_proofs.rs new file mode 100644 index 0000000000..79e6e081f1 --- /dev/null +++ b/crates/zk-prover/src/proof_request/effects/aggregation_proofs.rs @@ -0,0 +1,180 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Generate and publish aggregator C5 and C7 proofs. + +use super::*; + +impl ProofRequestActor { + /// Handle PkAggregationProofPending: dispatch C5 proof generation. + pub(in crate::actors::proof_request) fn handle_pk_aggregation_proof_pending( + &mut self, + msg: TypedEvent, + ) { + let (msg, ec) = msg.into_components(); + let e3_id = msg.e3_id.clone(); + + if self.pending_pk_aggregation.contains_key(&e3_id) { + warn!( + "Duplicate PkAggregationProofPending for E3 {} — ignoring", + e3_id + ); + return; + } + + self.pending_pk_aggregation.insert( + e3_id.clone(), + PendingPkAggregationProof { + ec: ec.clone(), + request: msg.proof_request.clone(), + }, + ); + + let correlation_id = CorrelationId::new(); + self.pk_aggregation_correlation + .insert(correlation_id, e3_id.clone()); + + info!("Requesting C5 PkAggregation proof for E3 {}", e3_id); + if let Err(err) = self.bus.publish( + ComputeRequest::zk( + ZkRequest::PkAggregation(msg.proof_request), + correlation_id, + e3_id.clone(), + ), + ec, + ) { + error!("Failed to publish C5 proof request: {err}"); + self.pk_aggregation_correlation.remove(&correlation_id); + self.pending_pk_aggregation.remove(&e3_id); + } + } + + /// Handle C5 proof response — sign proof and publish PkAggregationProofSigned. + pub(in crate::actors::proof_request) fn handle_pk_aggregation_proof_response( + &mut self, + correlation_id: &CorrelationId, + proof: Proof, + ) { + let Some(e3_id) = self.pk_aggregation_correlation.remove(correlation_id) else { + return; + }; + + let Some(pending) = self.pending_pk_aggregation.remove(&e3_id) else { + error!( + "No pending pk aggregation proof for E3 {} — orphan correlation", + e3_id + ); + return; + }; + + let Some(signed) = self.sign_proof(&e3_id, ProofType::C5PkAggregation, proof) else { + error!("Failed to sign C5 proof — PkAggregationProofSigned will not be published"); + self.fail_dkg_round(e3_id, &pending.ec, "C5 signing error"); + return; + }; + + info!( + "C5 proof signed for E3 {} (signer: {})", + e3_id, + self.signer.address() + ); + + if let Err(err) = self.bus.publish( + PkAggregationProofSigned { + e3_id: e3_id.clone(), + signed_proof: signed, + }, + pending.ec, + ) { + error!("Failed to publish PkAggregationProofSigned: {err}"); + } + } + + /// Handle AggregationProofPending: dispatch C7 proof generation. + pub(in crate::actors::proof_request) fn handle_aggregation_proof_pending( + &mut self, + msg: TypedEvent, + ) { + let (msg, ec) = msg.into_components(); + let e3_id = msg.e3_id.clone(); + + if self.pending_aggregation.contains_key(&e3_id) { + warn!( + "Duplicate AggregationProofPending for E3 {} — ignoring", + e3_id + ); + return; + } + + self.pending_aggregation + .insert(e3_id.clone(), PendingAggregationProof { ec: ec.clone() }); + + let correlation_id = CorrelationId::new(); + self.aggregation_correlation + .insert(correlation_id, e3_id.clone()); + + info!( + "Requesting C7 DecryptedSharesAggregation proof for E3 {}", + e3_id + ); + if let Err(err) = self.bus.publish( + ComputeRequest::zk( + ZkRequest::DecryptedSharesAggregation(msg.proof_request), + correlation_id, + e3_id.clone(), + ), + ec, + ) { + error!("Failed to publish C7 proof request: {err}"); + self.aggregation_correlation.remove(&correlation_id); + self.pending_aggregation.remove(&e3_id); + } + } + + /// Handle C7 proof response — sign proofs and publish AggregationProofSigned. + pub(in crate::actors::proof_request) fn handle_aggregation_proof_response( + &mut self, + correlation_id: &CorrelationId, + proofs: Vec, + ) { + let Some(e3_id) = self.aggregation_correlation.remove(correlation_id) else { + return; + }; + + let Some(pending) = self.pending_aggregation.remove(&e3_id) else { + error!( + "No pending aggregation proof for E3 {} — orphan correlation", + e3_id + ); + return; + }; + + // Sign each C7 proof + let mut signed_proofs = Vec::with_capacity(proofs.len()); + for proof in proofs { + let Some(signed) = + self.sign_proof(&e3_id, ProofType::C7DecryptedSharesAggregation, proof) + else { + error!("Failed to sign C7 proof — AggregationProofSigned will not be published"); + self.fail_decryption_round(e3_id.clone(), &pending.ec, "C7 signing error"); + return; + }; + signed_proofs.push(signed); + } + + info!( + "All C7 proofs signed for E3 {} (signer: {})", + e3_id, + self.signer.address() + ); + + if let Err(err) = self.bus.publish( + AggregationProofSigned { + e3_id: e3_id.clone(), + signed_proofs, + }, + pending.ec, + ) { + error!("Failed to publish AggregationProofSigned: {err}"); + } + } +} diff --git a/crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs b/crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs new file mode 100644 index 0000000000..1674c3c166 --- /dev/null +++ b/crates/zk-prover/src/proof_request/effects/decryption_key_proofs.rs @@ -0,0 +1,183 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Generate, correlate, sign, and publish C4 proof bundles. + +use super::*; + +impl ProofRequestActor { + /// Handle DecryptionShareProofsPending: dispatch C4 proof generation. + pub(in crate::actors::proof_request) fn handle_decryption_share_proofs_pending( + &mut self, + msg: TypedEvent, + ) { + let (msg, ec) = msg.into_components(); + let e3_id = msg.e3_id.clone(); + let esm_count = msg.esm_requests.len(); + + if self.pending_decryption.contains_key(&e3_id) { + warn!( + "Duplicate DecryptionShareProofsPending for E3 {} — ignoring", + e3_id + ); + return; + } + + self.pending_decryption.insert( + e3_id.clone(), + PendingDecryptionProofs { + party_id: msg.party_id, + node: msg.node, + ec: ec.clone(), + sk_proof: None, + esm_proofs: HashMap::new(), + expected_esm_count: esm_count, + }, + ); + + // C4a/C4b: dispatch share-decryption proof requests in canonical seq + // order. The pure domain planner owns seq assignment; the actor only + // allocates correlation ids, publishes, and rolls back on failure. + let c4_base_seq = self + .node_agg_meta + .get(&e3_id) + .map(NodeAggregationMeta::c4_base_seq) + .unwrap_or(0); + for item in plan_decryption_dispatch(msg.sk_request, msg.esm_requests, c4_base_seq) { + let corr = CorrelationId::new(); + self.decryption_correlation + .insert(corr, (e3_id.clone(), item.kind, item.seq)); + if let Err(err) = self.bus.publish( + ComputeRequest::zk(item.request, corr, e3_id.clone()), + ec.clone(), + ) { + error!("Failed to publish C4 proof request: {err}"); + self.decryption_correlation + .retain(|_, (eid, _, _)| *eid != e3_id); + self.pending_decryption.remove(&e3_id); + return; + } + } + } + + /// Handle a C4 proof response — store and check completeness. + pub(in crate::actors::proof_request) fn handle_decryption_proof_response( + &mut self, + correlation_id: &CorrelationId, + proof: Proof, + ec: &EventContext, + ) { + let Some((e3_id, kind, seq)) = self.decryption_correlation.remove(correlation_id) else { + return; + }; + + let Some(pending) = self.pending_decryption.get_mut(&e3_id) else { + error!( + "No pending decryption proofs for E3 {} — orphan correlation", + e3_id + ); + return; + }; + + let proof_for_agg = proof.clone(); + match kind { + DecryptionProofKind::SecretKey => { + info!("Received C4a SK decryption proof for E3 {}", e3_id); + pending.sk_proof = Some(proof); + } + DecryptionProofKind::SmudgingNoise { esi_idx } => { + info!( + "Received C4b ESM decryption proof [{}] for E3 {}", + esi_idx, e3_id + ); + pending.esm_proofs.insert(esi_idx, proof); + } + } + + if let Some(meta) = self.node_agg_meta.get(&e3_id) { + if meta.proof_aggregation_enabled { + if let Err(err) = self.bus.publish( + DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: meta.party_id, + proof: proof_for_agg, + seq, + }, + ec.clone(), + ) { + error!( + "Failed to publish DKGInnerProofReady for C4 seq={}: {err}", + seq + ); + } + } + } + + if pending.is_complete() { + info!( + "All C4 proofs complete for E3 {} — signing and publishing DecryptionKeyShared", + e3_id + ); + let pending = self.pending_decryption.remove(&e3_id).unwrap(); + self.sign_and_publish_decryption_key_shared(&e3_id, pending); + } + } + + /// Sign all C4 proofs and publish DecryptionKeyShared (Exchange #3). + pub(in crate::actors::proof_request) fn sign_and_publish_decryption_key_shared( + &mut self, + e3_id: &E3id, + pending: PendingDecryptionProofs, + ) { + // Sign C4a (SK decryption proof) + let Some(signed_sk) = self.sign_proof( + e3_id, + ProofType::C4aSkShareDecryption, + pending.sk_proof.expect("checked in is_complete"), + ) else { + error!("Failed to sign C4a SK proof — DecryptionKeyShared will not be published"); + self.fail_dkg_round(e3_id.clone(), &pending.ec, "C4a signing error"); + return; + }; + + // Sign C4b (ESM decryption proofs) in esi_idx order + let mut signed_esms = Vec::with_capacity(pending.expected_esm_count); + for idx in 0..pending.expected_esm_count { + let proof = pending + .esm_proofs + .get(&idx) + .expect("checked in is_complete") + .clone(); + let Some(signed) = self.sign_proof(e3_id, ProofType::C4bESmShareDecryption, proof) + else { + error!( + "Failed to sign C4b ESM proof [{}] — DecryptionKeyShared will not be published", + idx + ); + self.fail_dkg_round(e3_id.clone(), &pending.ec, "C4b signing error"); + return; + }; + signed_esms.push(signed); + } + + info!( + "All C4 proofs signed for E3 {} party {} (signer: {})", + e3_id, + pending.party_id, + self.signer.address() + ); + + if let Err(err) = self.bus.publish( + DecryptionKeyShared { + e3_id: e3_id.clone(), + party_id: pending.party_id, + node: pending.node, + signed_sk_decryption_proof: signed_sk, + signed_e_sm_decryption_proofs: signed_esms, + external: false, + }, + pending.ec, + ) { + error!("Failed to publish DecryptionKeyShared: {err}"); + } + } +} diff --git a/crates/zk-prover/src/proof_request/effects/decryption_share_proofs.rs b/crates/zk-prover/src/proof_request/effects/decryption_share_proofs.rs new file mode 100644 index 0000000000..8d6b381f47 --- /dev/null +++ b/crates/zk-prover/src/proof_request/effects/decryption_share_proofs.rs @@ -0,0 +1,121 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Generate and publish per-party C6 decryption-share proofs. + +use super::*; + +impl ProofRequestActor { + /// Handle ShareDecryptionProofPending: dispatch C6 proof generation. + pub(in crate::actors::proof_request) fn handle_share_decryption_proof_pending( + &mut self, + msg: TypedEvent, + ) { + let (msg, ec) = msg.into_components(); + let e3_id = msg.e3_id.clone(); + + if self.pending_share_decryption.contains_key(&e3_id) { + warn!( + "Duplicate ShareDecryptionProofPending for E3 {} — ignoring", + e3_id + ); + return; + } + + self.pending_share_decryption.insert( + e3_id.clone(), + PendingShareDecryptionProof { + party_id: msg.party_id, + node: msg.node, + decryption_share: msg.decryption_share, + ec: ec.clone(), + }, + ); + + let correlation_id = CorrelationId::new(); + self.share_decryption_correlation + .insert(correlation_id, e3_id.clone()); + + info!( + "Requesting C6 ThresholdShareDecryption proof for E3 {}", + e3_id + ); + if let Err(err) = self.bus.publish( + ComputeRequest::zk( + ZkRequest::ThresholdShareDecryption(msg.proof_request), + correlation_id, + e3_id.clone(), + ), + ec, + ) { + error!("Failed to publish C6 proof request: {err}"); + self.share_decryption_correlation.remove(&correlation_id); + self.pending_share_decryption.remove(&e3_id); + } + } + + /// Handle C6 proof response — sign proofs, publish DecryptionshareCreated. + pub(in crate::actors::proof_request) fn handle_share_decryption_proof_response( + &mut self, + correlation_id: &CorrelationId, + proofs: Vec, + ) { + let Some(e3_id) = self.share_decryption_correlation.remove(correlation_id) else { + return; + }; + + let Some(pending) = self.pending_share_decryption.remove(&e3_id) else { + error!( + "No pending share decryption proof for E3 {} — orphan correlation", + e3_id + ); + return; + }; + + // Sign raw C6 proofs (for ShareVerification) + let mut signed_proofs = Vec::with_capacity(proofs.len()); + for proof in proofs { + let Some(signed) = + self.sign_proof(&e3_id, ProofType::C6ThresholdShareDecryption, proof) + else { + error!("Failed to sign C6 proof — DecryptionshareCreated will not be published"); + self.fail_decryption_round(e3_id.clone(), &pending.ec, "C6 signing error"); + return; + }; + signed_proofs.push(signed); + } + + info!( + "All C6 proofs signed for E3 {} party {} (signer: {})", + e3_id, + pending.party_id, + self.signer.address() + ); + + let ec = pending.ec; + + match self.bus.publish( + DecryptionshareCreated { + party_id: pending.party_id, + node: pending.node, + e3_id: e3_id.clone(), + decryption_share: pending.decryption_share, + signed_decryption_proofs: signed_proofs, + }, + ec.clone(), + ) { + Ok(_) => { + if let Err(err) = self.bus.publish( + DecryptionShareProofSigned { + e3_id: e3_id.clone(), + }, + ec, + ) { + error!("Failed to publish DecryptionShareProofSigned: {err}"); + } + } + Err(err) => { + error!("Failed to publish DecryptionshareCreated: {err}"); + } + } + } +} diff --git a/crates/zk-prover/src/proof_request/effects/dkg_proofs.rs b/crates/zk-prover/src/proof_request/effects/dkg_proofs.rs new file mode 100644 index 0000000000..9b33d40302 --- /dev/null +++ b/crates/zk-prover/src/proof_request/effects/dkg_proofs.rs @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Dispatch C0 and threshold-share proof work and route compute results. + +use super::*; + +impl ProofRequestActor { + pub(in crate::actors::proof_request) fn handle_encryption_key_pending( + &mut self, + msg: TypedEvent, + ) { + let (msg, ec) = msg.into_components(); + let correlation_id = CorrelationId::new(); + self.pending.insert( + correlation_id, + PendingProofRequest { + e3_id: msg.e3_id.clone(), + key: msg.key.clone(), + }, + ); + + let request = ComputeRequest::zk( + ZkRequest::PkBfv(PkBfvProofRequest::new( + msg.key.pk_bfv.clone(), + msg.params_preset, + msg.committee_size, + )), + correlation_id, + msg.e3_id, + ); + + info!("Requesting C0 proof generation"); + if let Err(err) = self.bus.publish(request, ec) { + error!("Failed to publish ZK proof request: {err}"); + self.pending.remove(&correlation_id); + } + } + + pub(in crate::actors::proof_request) fn handle_threshold_share_pending( + &mut self, + msg: TypedEvent, + ) { + let (msg, ec) = msg.into_components(); + let e3_id = msg.e3_id.clone(); + + let sk_enc_count = msg.sk_share_encryption_requests.len(); + let e_sm_enc_count = msg.e_sm_share_encryption_requests.len(); + + let total_expected = NodeAggregationMeta::total_expected_for(sk_enc_count, e_sm_enc_count); + let pending_c0 = self + .node_agg_meta + .get(&e3_id) + .and_then(|m| m.pending_c0.clone()); + self.node_agg_meta.insert( + e3_id.clone(), + NodeAggregationMeta { + party_id: msg.full_share.party_id, + total_expected, + pending_c0: None, + proof_aggregation_enabled: msg.proof_aggregation_enabled, + }, + ); + // If C0 proof arrived before meta, emit DKGInnerProofReady now + if let Some(c0_proof) = pending_c0 { + if msg.proof_aggregation_enabled { + if let Err(err) = self.bus.publish( + DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: msg.full_share.party_id, + proof: c0_proof, + seq: 0, + }, + ec.clone(), + ) { + error!("Failed to publish DKGInnerProofReady for C0: {err}"); + } + } + } + + self.pending_threshold.insert( + e3_id.clone(), + PendingThresholdProofs::new( + e3_id.clone(), + msg.full_share.clone(), + ec.clone(), + sk_enc_count, + e_sm_enc_count, + msg.recipient_party_ids, + ), + ); + + // C1/C2/C3: dispatch threshold proof requests in canonical seq order. + // Sequencing/kind assignment lives in the pure domain planner; the actor + // only allocates correlation ids, publishes, and rolls back on failure. + for item in plan_threshold_dispatch( + msg.proof_request, + msg.sk_share_computation_request, + msg.e_sm_share_computation_request, + msg.sk_share_encryption_requests, + msg.e_sm_share_encryption_requests, + ) { + let corr = CorrelationId::new(); + self.threshold_correlation + .insert(corr, (e3_id.clone(), item.kind, item.seq)); + if let Err(err) = self.bus.publish( + ComputeRequest::zk(item.request, corr, e3_id.clone()), + ec.clone(), + ) { + error!("Failed to publish threshold proof request: {err}"); + self.threshold_correlation + .retain(|_, (eid, _, _)| *eid != e3_id); + self.pending_threshold.remove(&e3_id); + return; + } + } + } + + pub(in crate::actors::proof_request) fn handle_compute_response( + &mut self, + msg: TypedEvent, + ) { + let (msg, ec) = msg.into_components(); + match &msg.response { + ComputeResponseKind::Zk(ZkResponse::PkBfv(resp)) => { + self.handle_pk_bfv_response(&msg.correlation_id, resp.proof.clone(), &ec); + } + ComputeResponseKind::Zk(ZkResponse::PkGeneration(resp)) => { + self.handle_threshold_proof_response(&msg.correlation_id, resp.proof.clone(), &ec); + } + ComputeResponseKind::Zk(ZkResponse::ShareComputation(resp)) => { + self.handle_threshold_proof_response(&msg.correlation_id, resp.proof.clone(), &ec); + } + ComputeResponseKind::Zk(ZkResponse::ShareEncryption(resp)) => { + self.handle_threshold_proof_response(&msg.correlation_id, resp.proof.clone(), &ec); + } + ComputeResponseKind::Zk(ZkResponse::DkgShareDecryption(resp)) => { + // Try C4 decryption proof first, then fall back to C1/C2/C3 threshold + if self + .decryption_correlation + .contains_key(&msg.correlation_id) + { + self.handle_decryption_proof_response( + &msg.correlation_id, + resp.proof.clone(), + &ec, + ); + } else { + self.handle_threshold_proof_response( + &msg.correlation_id, + resp.proof.clone(), + &ec, + ); + } + } + ComputeResponseKind::Zk(ZkResponse::ThresholdShareDecryption(resp)) => { + self.handle_share_decryption_proof_response( + &msg.correlation_id, + resp.proofs.clone(), + ); + } + ComputeResponseKind::Zk(ZkResponse::PkAggregation(resp)) => { + self.handle_pk_aggregation_proof_response(&msg.correlation_id, resp.proof.clone()); + } + ComputeResponseKind::Zk(ZkResponse::DecryptedSharesAggregation(resp)) => { + self.handle_aggregation_proof_response(&msg.correlation_id, resp.proofs.clone()); + } + _ => {} + } + } +} diff --git a/crates/zk-prover/src/proof_request/effects/encryption_key_result.rs b/crates/zk-prover/src/proof_request/effects/encryption_key_result.rs new file mode 100644 index 0000000000..a821e54a0c --- /dev/null +++ b/crates/zk-prover/src/proof_request/effects/encryption_key_result.rs @@ -0,0 +1,118 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Apply a C0 proof result and emit the verified encryption-key artifact. + +use super::*; + +impl ProofRequestActor { + pub(in crate::actors::proof_request) fn handle_pk_bfv_response( + &mut self, + correlation_id: &CorrelationId, + proof: Proof, + ec: &EventContext, + ) { + let Some(pending) = self.pending.remove(correlation_id) else { + error!( + "Received PkBfv ComputeResponse with correlation_id {:?} but no matching pending request found.", + correlation_id + ); + return; + }; + + let e3_id = pending.e3_id.clone(); + + let mut key = (*pending.key).clone(); + key.proof = Some(proof.clone()); + + // Always sign the proof payload — unsigned proofs are not published + let payload = ProofPayload { + e3_id: e3_id.clone(), + proof_type: ProofType::C0PkBfv, + proof: proof.clone(), + }; + + match SignedProofPayload::sign(payload, &self.signer) { + Ok(signed) => { + info!( + "Signed C0 proof for party {} (signer: {})", + key.party_id, + self.signer.address() + ); + key.signed_payload = Some(signed); + } + Err(err) => { + error!("Failed to sign C0 proof payload: {err} — proof will not be published"); + self.fail_dkg_round(e3_id, ec, "C0 signing error"); + return; + } + } + + let local_party_id = key.party_id; + if let Err(err) = self.bus.publish( + EncryptionKeyCreated { + e3_id: e3_id.clone(), + key: Arc::new(key), + external: false, + }, + ec.clone(), + ) { + error!("Failed to publish EncryptionKeyCreated: {err}"); + } + + // Publish the local node's own C0 as ProofVerificationPassed so the + // CommitmentConsistencyChecker caches it. Without this, C3 proofs from + // other parties that encrypt under this node's pk would fail the C3→C0 + // consistency check (the local C0 target wouldn't exist in the cache). + { + let msg = ( + Bytes::copy_from_slice(&proof.data), + Bytes::copy_from_slice(&proof.public_signals), + ) + .abi_encode(); + let data_hash: [u8; 32] = keccak256(&msg).into(); + + if let Err(err) = self.bus.publish( + ProofVerificationPassed { + e3_id: e3_id.clone(), + party_id: local_party_id, + address: self.signer.address(), + proof_type: ProofType::C0PkBfv, + data_hash, + public_signals: proof.public_signals.clone(), + proof_data: proof.data.clone(), + }, + ec.clone(), + ) { + error!("Failed to publish local C0 ProofVerificationPassed: {err}"); + } + } + + // Emit DKGInnerProofReady for C0, or buffer if meta not yet available + if let Some(meta) = self.node_agg_meta.get(&e3_id) { + if meta.proof_aggregation_enabled { + if let Err(err) = self.bus.publish( + DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: meta.party_id, + proof: proof.clone(), + seq: 0, + }, + ec.clone(), + ) { + error!("Failed to publish DKGInnerProofReady for C0: {err}"); + } + } + } else { + // ThresholdSharePending hasn't arrived yet — buffer C0 proof + self.node_agg_meta.insert( + e3_id.clone(), + NodeAggregationMeta { + party_id: 0, + total_expected: 0, + pending_c0: Some(proof), + proof_aggregation_enabled: true, // will be overwritten by ThresholdSharePending + }, + ); + } + } +} diff --git a/crates/zk-prover/src/proof_request/effects/failures.rs b/crates/zk-prover/src/proof_request/effects/failures.rs new file mode 100644 index 0000000000..9f2def6704 --- /dev/null +++ b/crates/zk-prover/src/proof_request/effects/failures.rs @@ -0,0 +1,84 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Correlate worker failures to the affected proof workflow. + +use super::*; + +impl ProofRequestActor { + pub(in crate::actors::proof_request) fn handle_compute_request_error( + &mut self, + msg: TypedEvent, + ) { + let (msg, ec) = msg.into_components(); + let ComputeRequestErrorKind::Zk(err) = msg.get_err() else { + return; + }; + + if let Some(pending) = self.pending.remove(msg.correlation_id()) { + error!( + "C0 proof request failed for E3 {}: {err} — key will not be published without proof", + pending.e3_id + ); + self.fail_dkg_round(pending.e3_id, &ec, "C0 proof request error"); + return; + } + + if let Some((e3_id, kind, _seq)) = self.threshold_correlation.remove(msg.correlation_id()) { + error!( + "DKG {:?} proof request failed for E3 {}: {err} — threshold share will not be published without proof", + kind, e3_id + ); + self.threshold_correlation + .retain(|_, (eid, _, _)| *eid != e3_id); + self.pending_threshold.remove(&e3_id); + self.fail_dkg_round(e3_id, &ec, "DKG threshold proof request error"); + return; + } + + if let Some((e3_id, kind, _seq)) = self.decryption_correlation.remove(msg.correlation_id()) + { + error!( + "C4 {:?} proof request failed for E3 {}: {err} — DecryptionKeyShared will not be published", + kind, e3_id + ); + self.decryption_correlation + .retain(|_, (eid, _, _)| *eid != e3_id); + self.pending_decryption.remove(&e3_id); + self.fail_dkg_round(e3_id, &ec, "C4 proof request error"); + return; + } + + if let Some(e3_id) = self + .share_decryption_correlation + .remove(msg.correlation_id()) + { + error!( + "C6 proof request failed for E3 {}: {err} — DecryptionshareCreated will not be published", + e3_id + ); + self.pending_share_decryption.remove(&e3_id); + self.fail_decryption_round(e3_id, &ec, "C6 proof request error"); + return; + } + + if let Some(e3_id) = self.pk_aggregation_correlation.remove(msg.correlation_id()) { + error!( + "C5 proof request failed for E3 {}: {err} — PkAggregationProofSigned will not be published", + e3_id + ); + self.pending_pk_aggregation.remove(&e3_id); + + self.fail_dkg_round(e3_id, &ec, "C5 proof request error"); + return; + } + + if let Some(e3_id) = self.aggregation_correlation.remove(msg.correlation_id()) { + error!( + "C7 proof request failed for E3 {}: {err} — AggregationProofSigned will not be published", + e3_id + ); + self.pending_aggregation.remove(&e3_id); + self.fail_decryption_round(e3_id, &ec, "C7 proof request error"); + } + } +} diff --git a/crates/zk-prover/src/proof_request/effects/mod.rs b/crates/zk-prover/src/proof_request/effects/mod.rs new file mode 100644 index 0000000000..4884a4c1e7 --- /dev/null +++ b/crates/zk-prover/src/proof_request/effects/mod.rs @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Proof-request effect execution grouped by business action. + +use super::*; + +mod aggregation_proofs; +mod decryption_key_proofs; +mod decryption_share_proofs; +mod dkg_proofs; +mod encryption_key_result; +mod failures; +mod publish_threshold_shares; +mod signing; diff --git a/crates/zk-prover/src/proof_request/effects/publish_threshold_shares.rs b/crates/zk-prover/src/proof_request/effects/publish_threshold_shares.rs new file mode 100644 index 0000000000..b5cd97d285 --- /dev/null +++ b/crates/zk-prover/src/proof_request/effects/publish_threshold_shares.rs @@ -0,0 +1,200 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Assemble and publish a complete signed C1/C2/C3 threshold-share bundle. + +use super::*; + +impl ProofRequestActor { + pub(in crate::actors::proof_request) fn publish_threshold_share_with_proofs( + &mut self, + pending: PendingThresholdProofs, + ) { + let e3_id = &pending.e3_id; + let party_id = pending.full_share.party_id; + let ec = &pending.ec; + + // Sign C1 (PkGeneration) + let Some(signed_pk_gen) = self.sign_proof( + e3_id, + ProofType::C1PkGeneration, + pending.pk_generation_proof.expect("checked"), + ) else { + error!("Failed to sign C1 proof — shares will not be published"); + self.fail_dkg_round(e3_id.clone(), ec, "C1 signing error"); + return; + }; + + // Sign C2a (SkShareComputation) + let Some(signed_c2a) = self.sign_proof( + e3_id, + ProofType::C2aSkShareComputation, + pending.sk_share_computation_proof.expect("checked"), + ) else { + error!("Failed to sign C2a proof — shares will not be published"); + self.fail_dkg_round(e3_id.clone(), ec, "C2a signing error"); + return; + }; + + // Sign C2b (ESmShareComputation) + let Some(signed_c2b) = self.sign_proof( + e3_id, + ProofType::C2bESmShareComputation, + pending.e_sm_share_computation_proof.expect("checked"), + ) else { + error!("Failed to sign C2b proof — shares will not be published"); + self.fail_dkg_round(e3_id.clone(), ec, "C2b signing error"); + return; + }; + + let Some(signed_c3a_map) = self.sign_and_group_proofs( + e3_id, + ProofType::C3aSkShareEncryption, + pending + .sk_share_encryption_proofs + .iter() + .map(|((recipient, _row), proof)| (*recipient, proof.clone())), + ) else { + error!("Failed to sign C3a proofs — shares will not be published"); + self.fail_dkg_round(e3_id.clone(), ec, "C3a signing error"); + return; + }; + + let Some(signed_c3b_map) = self.sign_and_group_proofs( + e3_id, + ProofType::C3bESmShareEncryption, + pending + .e_sm_share_encryption_proofs + .iter() + .map(|((_esi, recipient, _row), proof)| (*recipient, proof.clone())), + ) else { + error!("Failed to sign C3b proofs — shares will not be published"); + self.fail_dkg_round(e3_id.clone(), ec, "C3b signing error"); + return; + }; + + info!( + "All proofs signed for E3 {} party {} (signer: {})", + e3_id, + party_id, + self.signer.address() + ); + + // Publish local proof events for the node's own state tracking + if let Err(err) = self.bus.publish( + PkGenerationProofSigned { + e3_id: e3_id.clone(), + party_id, + signed_proof: signed_pk_gen, + }, + ec.clone(), + ) { + error!("Failed to publish PkGenerationProofSigned: {err}"); + } + + if let Err(err) = self.bus.publish( + DkgProofSigned { + e3_id: e3_id.clone(), + party_id, + signed_proof: signed_c2a.clone(), + }, + ec.clone(), + ) { + error!("Failed to publish SkDkgProofSigned: {err}"); + } + + if let Err(err) = self.bus.publish( + DkgProofSigned { + e3_id: e3_id.clone(), + party_id, + signed_proof: signed_c2b.clone(), + }, + ec.clone(), + ) { + error!("Failed to publish ESmDkgProofSigned: {err}"); + } + + // Publish C3a signed proofs (reuse already-signed proofs from signed_c3a_map) + for signed_proofs in signed_c3a_map.values() { + for signed in signed_proofs { + if let Err(err) = self.bus.publish( + DkgProofSigned { + e3_id: e3_id.clone(), + party_id, + signed_proof: signed.clone(), + }, + ec.clone(), + ) { + error!("Failed to publish SkShareEncryptionProofSigned: {err}"); + } + } + } + + // Publish C3b signed proofs (reuse already-signed proofs from signed_c3b_map) + for signed_proofs in signed_c3b_map.values() { + for signed in signed_proofs { + if let Err(err) = self.bus.publish( + DkgProofSigned { + e3_id: e3_id.clone(), + party_id, + signed_proof: signed.clone(), + }, + ec.clone(), + ) { + error!("Failed to publish ESmShareEncryptionProofSigned: {err}"); + } + } + } + + // Publish ThresholdShareCreated with proofs attached for each recipient + let share = &pending.full_share; + let num_parties = share.num_parties(); + + info!( + "Publishing ThresholdShareCreated for E3 {} to {} parties", + e3_id, num_parties + ); + + for (positional_idx, &real_party_id) in pending.recipient_party_ids.iter().enumerate() { + match share.extract_for_party(positional_idx) { + Some(party_share) => { + let proof_key = real_party_id as usize; + let c3a_proofs = signed_c3a_map.get(&proof_key).cloned().unwrap_or_default(); + let c3b_proofs = signed_c3b_map.get(&proof_key).cloned().unwrap_or_default(); + + if let Err(err) = self.bus.publish( + ThresholdShareCreated { + e3_id: e3_id.clone(), + share: Arc::new(party_share), + target_party_id: real_party_id, + external: false, + signed_c2a_proof: Some(signed_c2a.clone()), + signed_c2b_proof: Some(signed_c2b.clone()), + signed_c3a_proofs: c3a_proofs, + signed_c3b_proofs: c3b_proofs, + }, + ec.clone(), + ) { + error!( + "Failed to publish ThresholdShareCreated for party {} (idx {}): {err}", + real_party_id, positional_idx + ); + } + } + None if real_party_id == party_id => { + // Own slot is sparse (no self-encryption); nothing to publish. + trace!( + "Skipping ThresholdShareCreated for own slot (party {} idx {})", + real_party_id, + positional_idx + ); + } + None => { + error!( + "Missing encrypted share for recipient party {} (idx {}) from sender party {}; ThresholdShareCreated will not be published for that recipient", + real_party_id, positional_idx, party_id + ); + } + } + } + } +} diff --git a/crates/zk-prover/src/proof_request/effects/signing.rs b/crates/zk-prover/src/proof_request/effects/signing.rs new file mode 100644 index 0000000000..8f2704677a --- /dev/null +++ b/crates/zk-prover/src/proof_request/effects/signing.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Shared proof signing, failure publication, and grouping helpers. + +use super::*; + +impl ProofRequestActor { + pub(in crate::actors::proof_request) fn handle_threshold_proof_response( + &mut self, + correlation_id: &CorrelationId, + proof: Proof, + ec: &EventContext, + ) { + let Some((e3_id, kind, seq)) = self.threshold_correlation.remove(correlation_id) else { + return; + }; + + let Some(pending) = self.pending_threshold.get_mut(&e3_id) else { + error!( + "No pending threshold proofs for E3 {} — orphan correlation", + e3_id + ); + return; + }; + + let proof_for_agg = proof.clone(); + pending.store_proof(&kind, proof); + info!( + "Received {:?} proof for E3 {} ({}/{})", + kind, + e3_id, + pending.total_received(), + pending.total_expected() + ); + + if let Some(meta) = self.node_agg_meta.get(&e3_id) { + if meta.proof_aggregation_enabled { + if let Err(err) = self.bus.publish( + DKGInnerProofReady { + e3_id: e3_id.clone(), + party_id: meta.party_id, + proof: proof_for_agg, + seq, + }, + ec.clone(), + ) { + error!( + "Failed to publish DKGInnerProofReady for {:?} seq={}: {err}", + kind, seq + ); + } + } + } + + if pending.is_complete() { + info!( + "All {} threshold proofs complete for E3 {}", + pending.total_expected(), + e3_id + ); + let pending = self.pending_threshold.remove(&e3_id).unwrap(); + self.publish_threshold_share_with_proofs(pending); + } + } + + pub(in crate::actors::proof_request) fn sign_proof( + &self, + e3_id: &E3id, + proof_type: ProofType, + proof: Proof, + ) -> Option { + let payload = ProofPayload { + e3_id: e3_id.clone(), + proof_type, + proof, + }; + match SignedProofPayload::sign(payload, &self.signer) { + Ok(signed) => Some(signed), + Err(err) => { + error!("Failed to sign {:?} proof: {err}", proof_type); + None + } + } + } + + pub(in crate::actors::proof_request) fn fail_dkg_round( + &self, + e3_id: E3id, + ec: &EventContext, + context: &str, + ) { + if let Err(err) = self.bus.publish( + E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CommitteeFinalized, + reason: FailureReason::DKGInvalidShares, + }, + ec.clone(), + ) { + error!( + "Failed to publish E3Failed for {context} on E3 {}: {err}", + e3_id + ); + } + } + + pub(in crate::actors::proof_request) fn fail_decryption_round( + &self, + e3_id: E3id, + ec: &EventContext, + context: &str, + ) { + if let Err(err) = self.bus.publish( + E3Failed { + e3_id: e3_id.clone(), + failed_at_stage: E3Stage::CiphertextReady, + reason: FailureReason::DecryptionInvalidShares, + }, + ec.clone(), + ) { + error!( + "Failed to publish E3Failed for {context} on E3 {}: {err}", + e3_id + ); + } + } + + pub(in crate::actors::proof_request) fn sign_and_group_proofs( + &self, + e3_id: &E3id, + proof_type: ProofType, + proofs: impl Iterator, + ) -> Option>> { + let mut map: BTreeMap> = BTreeMap::new(); + for (recipient, proof) in proofs { + let signed = self.sign_proof(e3_id, proof_type, proof)?; + map.entry(recipient).or_default().push(signed); + } + Some(map) + } +} diff --git a/crates/zk-prover/src/proof_request/handlers.rs b/crates/zk-prover/src/proof_request/handlers.rs new file mode 100644 index 0000000000..2197f9194f --- /dev/null +++ b/crates/zk-prover/src/proof_request/handlers.rs @@ -0,0 +1,141 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Actix envelope routing for proof-generation workflows. + +use super::*; + +impl Actor for ProofRequestActor { + type Context = Context; +} + +impl Handler for ProofRequestActor { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + + match msg { + InterfoldEventData::EncryptionKeyPending(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ThresholdSharePending(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ComputeResponse(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ComputeRequestError(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::DecryptionShareProofsPending(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ShareDecryptionProofPending(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::PkAggregationProofPending(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::AggregationProofPending(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + _ => (), + } + } +} + +impl Handler> for ProofRequestActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_encryption_key_pending(msg) + } +} + +impl Handler> for ProofRequestActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_threshold_share_pending(msg); + } +} + +impl Handler> for ProofRequestActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_compute_response(msg) + } +} + +impl Handler> for ProofRequestActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_compute_request_error(msg) + } +} + +impl Handler> for ProofRequestActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_decryption_share_proofs_pending(msg) + } +} + +impl Handler> for ProofRequestActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_share_decryption_proof_pending(msg) + } +} + +impl Handler> for ProofRequestActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_pk_aggregation_proof_pending(msg) + } +} + +impl Handler> for ProofRequestActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_aggregation_proof_pending(msg) + } +} diff --git a/crates/zk-prover/src/proof_request/state.rs b/crates/zk-prover/src/proof_request/state.rs new file mode 100644 index 0000000000..54c9253bf0 --- /dev/null +++ b/crates/zk-prover/src/proof_request/state.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Per-E3 pending proof state and canonical proof identifiers. + +use super::*; + +/// Identifies which threshold (C1/C2/C3) proof a response corresponds to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum ThresholdProofKind { + PkGeneration, + SkShareComputation, + ESmShareComputation, + SkShareEncryption { + recipient_party_id: usize, + row_index: usize, + }, + ESmShareEncryption { + esi_index: usize, + recipient_party_id: usize, + row_index: usize, + }, +} + +/// Identifies which C4 (DkgShareDecryption) proof a response corresponds to. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) enum DecryptionProofKind { + SecretKey, + SmudgingNoise { esi_idx: usize }, +} + +/// Per-E3 metadata for streaming DKG inner proof aggregation. +#[derive(Clone, Debug)] +pub(crate) struct NodeAggregationMeta { + pub(crate) party_id: u64, + pub(crate) total_expected: usize, + /// Buffered C0 proof, if it arrived before meta was stored. + pub(crate) pending_c0: Option, + /// When false, skip emitting DKGInnerProofReady (no recursive DKG aggregation). + pub(crate) proof_aggregation_enabled: bool, +} + +impl NodeAggregationMeta { + /// Total expected inner proofs for streaming aggregation: + /// C0..C4 (4 + sk + esm + 2). Mirrors the node-fold collector sizing. + pub(crate) fn total_expected_for(sk_enc_count: usize, e_sm_enc_count: usize) -> usize { + 4 + sk_enc_count + e_sm_enc_count + 2 + } + + /// Base `seq` for the first C4 proof: just after all C0..C3 proofs. + pub(crate) fn c4_base_seq(&self) -> usize { + self.total_expected.saturating_sub(2) + } +} + +#[derive(Clone, Debug)] +pub(crate) struct PendingProofRequest { + pub(crate) e3_id: E3id, + pub(crate) key: Arc, +} + +#[derive(Clone, Debug)] +pub(crate) struct PendingThresholdProofs { + pub(crate) e3_id: E3id, + pub(crate) full_share: Arc, + pub(crate) ec: EventContext, + pub(crate) pk_generation_proof: Option, + pub(crate) sk_share_computation_proof: Option, + pub(crate) e_sm_share_computation_proof: Option, + /// C3a proofs: keyed by (recipient_party_id, row_index) + pub(crate) sk_share_encryption_proofs: HashMap<(usize, usize), Proof>, + pub(crate) expected_sk_enc_count: usize, + /// C3b proofs: keyed by (esi_index, recipient_party_id, row_index) + pub(crate) e_sm_share_encryption_proofs: HashMap<(usize, usize, usize), Proof>, + pub(crate) expected_e_sm_enc_count: usize, + /// Maps positional index to real party_id (from ThresholdSharePending). + pub(crate) recipient_party_ids: Vec, +} + +impl PendingThresholdProofs { + pub(crate) fn new( + e3_id: E3id, + full_share: Arc, + ec: EventContext, + expected_sk_enc_count: usize, + expected_e_sm_enc_count: usize, + recipient_party_ids: Vec, + ) -> Self { + Self { + e3_id, + full_share, + ec, + pk_generation_proof: None, + sk_share_computation_proof: None, + e_sm_share_computation_proof: None, + sk_share_encryption_proofs: HashMap::new(), + expected_sk_enc_count, + e_sm_share_encryption_proofs: HashMap::new(), + expected_e_sm_enc_count, + recipient_party_ids, + } + } + + pub(crate) fn is_complete(&self) -> bool { + self.pk_generation_proof.is_some() + && self.sk_share_computation_proof.is_some() + && self.e_sm_share_computation_proof.is_some() + && self.sk_share_encryption_proofs.len() == self.expected_sk_enc_count + && self.e_sm_share_encryption_proofs.len() == self.expected_e_sm_enc_count + } + + pub(crate) fn store_proof(&mut self, kind: &ThresholdProofKind, proof: Proof) { + match kind { + ThresholdProofKind::PkGeneration => self.pk_generation_proof = Some(proof), + ThresholdProofKind::SkShareComputation => self.sk_share_computation_proof = Some(proof), + ThresholdProofKind::ESmShareComputation => { + self.e_sm_share_computation_proof = Some(proof) + } + ThresholdProofKind::SkShareEncryption { + recipient_party_id, + row_index, + } => { + self.sk_share_encryption_proofs + .insert((*recipient_party_id, *row_index), proof); + } + ThresholdProofKind::ESmShareEncryption { + esi_index, + recipient_party_id, + row_index, + } => { + self.e_sm_share_encryption_proofs + .insert((*esi_index, *recipient_party_id, *row_index), proof); + } + } + } + + pub(crate) fn total_expected(&self) -> usize { + 3 + self.expected_sk_enc_count + self.expected_e_sm_enc_count + } + + pub(crate) fn total_received(&self) -> usize { + let base = [ + self.pk_generation_proof.is_some(), + self.sk_share_computation_proof.is_some(), + self.e_sm_share_computation_proof.is_some(), + ] + .iter() + .filter(|&&v| v) + .count(); + base + self.sk_share_encryption_proofs.len() + self.e_sm_share_encryption_proofs.len() + } +} + +/// Pending C4 (DkgShareDecryption) proof generation state. +#[derive(Clone, Debug)] +pub(crate) struct PendingDecryptionProofs { + pub(crate) party_id: u64, + pub(crate) node: String, + pub(crate) ec: EventContext, + pub(crate) sk_proof: Option, + pub(crate) esm_proofs: HashMap, + pub(crate) expected_esm_count: usize, +} + +impl PendingDecryptionProofs { + pub(crate) fn is_complete(&self) -> bool { + self.sk_proof.is_some() + && self.esm_proofs.len() == self.expected_esm_count + && (0..self.expected_esm_count).all(|i| self.esm_proofs.contains_key(&i)) + } +} + +/// Pending C5 (PkAggregation) proof generation state. +#[derive(Clone, Debug)] +pub(crate) struct PendingPkAggregationProof { + pub(crate) ec: EventContext, + #[allow(dead_code)] + pub(crate) request: PkAggregationProofRequest, +} + +/// Pending C6 (ShareDecryptionProof) proof generation state. +#[derive(Clone, Debug)] +pub(crate) struct PendingShareDecryptionProof { + pub(crate) party_id: u64, + pub(crate) node: String, + pub(crate) decryption_share: Vec, + pub(crate) ec: EventContext, +} + +/// Pending C7 (DecryptedSharesAggregation) proof generation state. +#[derive(Clone, Debug)] +pub(crate) struct PendingAggregationProof { + pub(crate) ec: EventContext, +} diff --git a/crates/zk-prover/src/proof_request/transitions.rs b/crates/zk-prover/src/proof_request/transitions.rs new file mode 100644 index 0000000000..9e404c0228 --- /dev/null +++ b/crates/zk-prover/src/proof_request/transitions.rs @@ -0,0 +1,101 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Deterministic C1-C4 dispatch planning and sequence assignment. + +use super::*; + +/// A single planned threshold (C1/C2/C3) proof request: which proof, its `seq` +/// index for streaming aggregation, and the [`ZkRequest`] to dispatch. +pub(crate) struct ThresholdDispatchItem { + pub(crate) kind: ThresholdProofKind, + pub(crate) seq: usize, + pub(crate) request: ZkRequest, +} + +/// Build the deterministic, ordered set of C1/C2/C3 proof requests for a +/// `ThresholdSharePending` event. Pure: assigns the canonical `seq` indices +/// (C1=1, C2a=2, C2b=3, C3a[i]=4+i, C3b[j]=4+sk_count+j) and wraps each request. +pub(crate) fn plan_threshold_dispatch( + proof_request: PkGenerationProofRequest, + sk_share_computation_request: ShareComputationProofRequest, + e_sm_share_computation_request: ShareComputationProofRequest, + sk_share_encryption_requests: Vec, + e_sm_share_encryption_requests: Vec, +) -> Vec { + let sk_enc_count = sk_share_encryption_requests.len(); + let mut items = Vec::with_capacity(3 + sk_enc_count + e_sm_share_encryption_requests.len()); + + items.push(ThresholdDispatchItem { + kind: ThresholdProofKind::PkGeneration, + seq: 1, + request: ZkRequest::PkGeneration(proof_request), + }); + items.push(ThresholdDispatchItem { + kind: ThresholdProofKind::SkShareComputation, + seq: 2, + request: ZkRequest::ShareComputation(sk_share_computation_request), + }); + items.push(ThresholdDispatchItem { + kind: ThresholdProofKind::ESmShareComputation, + seq: 3, + request: ZkRequest::ShareComputation(e_sm_share_computation_request), + }); + + for (i, req) in sk_share_encryption_requests.into_iter().enumerate() { + let kind = ThresholdProofKind::SkShareEncryption { + recipient_party_id: req.recipient_party_id, + row_index: req.row_index, + }; + items.push(ThresholdDispatchItem { + kind, + seq: 4 + i, + request: ZkRequest::ShareEncryption(req), + }); + } + + for (j, req) in e_sm_share_encryption_requests.into_iter().enumerate() { + let kind = ThresholdProofKind::ESmShareEncryption { + esi_index: req.esi_index, + recipient_party_id: req.recipient_party_id, + row_index: req.row_index, + }; + items.push(ThresholdDispatchItem { + kind, + seq: 4 + sk_enc_count + j, + request: ZkRequest::ShareEncryption(req), + }); + } + + items +} + +/// A single planned C4 (DkgShareDecryption) proof request. +pub(crate) struct DecryptionDispatchItem { + pub(crate) kind: DecryptionProofKind, + pub(crate) seq: usize, + pub(crate) request: ZkRequest, +} + +/// Build the ordered set of C4 proof requests (SecretKey then SmudgingNoise[i]). +/// `c4_base_seq` is the streaming-aggregation `seq` of the C4a (SecretKey) proof; +/// each C4b proof follows at `c4_base_seq + 1 + esi_idx`. +pub(crate) fn plan_decryption_dispatch( + sk_request: DkgShareDecryptionProofRequest, + esm_requests: Vec, + c4_base_seq: usize, +) -> Vec { + let mut items = Vec::with_capacity(1 + esm_requests.len()); + items.push(DecryptionDispatchItem { + kind: DecryptionProofKind::SecretKey, + seq: c4_base_seq, + request: ZkRequest::DkgShareDecryption(sk_request), + }); + for (esi_idx, esm_req) in esm_requests.into_iter().enumerate() { + items.push(DecryptionDispatchItem { + kind: DecryptionProofKind::SmudgingNoise { esi_idx }, + seq: c4_base_seq + 1 + esi_idx, + request: ZkRequest::DkgShareDecryption(esm_req), + }); + } + items +} diff --git a/crates/zk-prover/src/proof_request/workflow.rs b/crates/zk-prover/src/proof_request/workflow.rs new file mode 100644 index 0000000000..c10bde14a5 --- /dev/null +++ b/crates/zk-prover/src/proof_request/workflow.rs @@ -0,0 +1,40 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Deterministic proof-request workflow dispatch and completion. +//! +//! The [`crate::actors::proof_request::ProofRequestActor`] is a thin transport +//! shell: it owns the event bus and signer and performs all publish/sign I/O. +//! This module owns the business logic — the per-E3 pending-proof state machines +//! (which proofs have arrived, when a set is complete) and the deterministic +//! dispatch *planning* (which proof requests to emit, in what order, with which +//! `seq` index). It has NO actix / `BusHandle` / signing concerns. + +use std::collections::HashMap; +use std::sync::Arc; + +use e3_events::{ + DkgShareDecryptionProofRequest, E3id, EncryptionKey, EventContext, PkAggregationProofRequest, + PkGenerationProofRequest, Proof, Sequenced, ShareComputationProofRequest, + ShareEncryptionProofRequest, ThresholdShare, ZkRequest, +}; +use e3_utils::utility_types::ArcBytes; + +#[path = "state.rs"] +mod state; +#[path = "transitions.rs"] +mod transitions; + +pub(crate) use state::{ + DecryptionProofKind, NodeAggregationMeta, PendingAggregationProof, PendingDecryptionProofs, + PendingPkAggregationProof, PendingProofRequest, PendingShareDecryptionProof, + PendingThresholdProofs, ThresholdProofKind, +}; +pub(crate) use transitions::{plan_decryption_dispatch, plan_threshold_dispatch}; + +#[cfg(test)] +#[path = "workflow_tests.rs"] +mod tests; diff --git a/crates/zk-prover/src/proof_request/workflow_tests.rs b/crates/zk-prover/src/proof_request/workflow_tests.rs new file mode 100644 index 0000000000..b9960f899d --- /dev/null +++ b/crates/zk-prover/src/proof_request/workflow_tests.rs @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use e3_crypto::SensitiveBytes; +use e3_events::CircuitName; +use e3_fhe_params::BfvPreset; +use e3_trbfv::shares::BfvEncryptedShares; +use e3_zk_helpers::{computation::DkgInputType, CiphernodesCommitteeSize}; + +fn ec() -> EventContext { + use e3_events::{InterfoldEventData, TestEvent, Unsequenced}; + EventContext::::from(InterfoldEventData::from(TestEvent::new("x", 0))).sequence(0) +} + +fn sensitive() -> SensitiveBytes { + SensitiveBytes::from_encrypted(&[]) +} + +fn full_share() -> Arc { + Arc::new(ThresholdShare { + party_id: 0, + pk_share: ArcBytes::from_bytes(&[]), + sk_sss: BfvEncryptedShares::default(), + esi_sss: vec![], + }) +} + +fn share_computation_req() -> ShareComputationProofRequest { + ShareComputationProofRequest { + secret_raw: sensitive(), + secret_sss_raw: sensitive(), + dkg_input_type: DkgInputType::SecretKey, + params_preset: BfvPreset::default(), + committee_size: CiphernodesCommitteeSize::Micro, + } +} + +fn pk_generation_req() -> PkGenerationProofRequest { + PkGenerationProofRequest { + pk0_share: ArcBytes::from_bytes(&[]), + sk: sensitive(), + eek: sensitive(), + e_sm: sensitive(), + params_preset: BfvPreset::default(), + committee_size: CiphernodesCommitteeSize::Micro, + } +} + +fn share_encryption_req( + recipient_party_id: usize, + row_index: usize, + esi_index: usize, +) -> ShareEncryptionProofRequest { + ShareEncryptionProofRequest { + share_row_raw: sensitive(), + ciphertext_raw: ArcBytes::from_bytes(&[]), + recipient_pk_raw: ArcBytes::from_bytes(&[]), + u_rns_raw: sensitive(), + e0_rns_raw: sensitive(), + e1_rns_raw: sensitive(), + dkg_input_type: DkgInputType::SecretKey, + params_preset: BfvPreset::default(), + committee_size: CiphernodesCommitteeSize::Micro, + recipient_party_id, + row_index, + esi_index, + } +} + +fn dkg_share_decryption_req() -> DkgShareDecryptionProofRequest { + DkgShareDecryptionProofRequest { + sk_bfv: sensitive(), + honest_ciphertexts_raw: vec![], + num_honest_parties: 0, + num_moduli: 0, + own_plaintext_idx: 0, + own_share_raw: sensitive(), + dkg_input_type: DkgInputType::SecretKey, + params_preset: BfvPreset::default(), + committee_size: CiphernodesCommitteeSize::Micro, + } +} + +fn proof(seed: u8) -> Proof { + Proof::new( + CircuitName::PkAggregation, + ArcBytes::from_bytes(&[seed]), + ArcBytes::from_bytes(&[seed.wrapping_add(1)]), + ) +} + +fn pending(sk: usize, esm: usize) -> PendingThresholdProofs { + PendingThresholdProofs::new( + E3id::new("1", 1), + full_share(), + ec(), + sk, + esm, + vec![1, 2, 3], + ) +} + +#[test] +fn threshold_completes_only_when_all_proofs_present() { + let mut p = pending(1, 1); + assert!(!p.is_complete()); + assert_eq!(p.total_expected(), 3 + 1 + 1); + assert_eq!(p.total_received(), 0); + + p.store_proof(&ThresholdProofKind::PkGeneration, proof(1)); + p.store_proof(&ThresholdProofKind::SkShareComputation, proof(2)); + p.store_proof(&ThresholdProofKind::ESmShareComputation, proof(3)); + assert!(!p.is_complete()); + assert_eq!(p.total_received(), 3); + + p.store_proof( + &ThresholdProofKind::SkShareEncryption { + recipient_party_id: 2, + row_index: 0, + }, + proof(4), + ); + assert!(!p.is_complete()); + p.store_proof( + &ThresholdProofKind::ESmShareEncryption { + esi_index: 0, + recipient_party_id: 2, + row_index: 0, + }, + proof(5), + ); + assert!(p.is_complete()); + assert_eq!(p.total_received(), 5); +} + +#[test] +fn store_proof_dedupes_by_key() { + let mut p = pending(2, 0); + let key = ThresholdProofKind::SkShareEncryption { + recipient_party_id: 2, + row_index: 0, + }; + p.store_proof(&key, proof(4)); + p.store_proof(&key, proof(9)); // same (recipient,row) overwrites + assert_eq!(p.sk_share_encryption_proofs.len(), 1); + assert!(!p.is_complete()); // still expecting 2 distinct sk enc proofs +} + +#[test] +fn decryption_completes_when_sk_and_all_esm_present() { + let mut d = PendingDecryptionProofs { + party_id: 7, + node: "n".into(), + ec: ec(), + sk_proof: None, + esm_proofs: HashMap::new(), + expected_esm_count: 2, + }; + assert!(!d.is_complete()); + d.sk_proof = Some(proof(1)); + d.esm_proofs.insert(0, proof(2)); + assert!(!d.is_complete()); + d.esm_proofs.insert(1, proof(3)); + assert!(d.is_complete()); +} + +#[test] +fn decryption_requires_contiguous_esm_indices() { + let mut d = PendingDecryptionProofs { + party_id: 7, + node: "n".into(), + ec: ec(), + sk_proof: Some(proof(1)), + esm_proofs: HashMap::new(), + expected_esm_count: 2, + }; + // Two entries but indices {0,2} — count matches but index 1 missing. + d.esm_proofs.insert(0, proof(2)); + d.esm_proofs.insert(2, proof(3)); + assert!(!d.is_complete()); +} + +#[test] +fn node_agg_meta_seq_helpers() { + assert_eq!(NodeAggregationMeta::total_expected_for(2, 1), 4 + 2 + 1 + 2); + let meta = NodeAggregationMeta { + party_id: 0, + total_expected: NodeAggregationMeta::total_expected_for(2, 1), + pending_c0: None, + proof_aggregation_enabled: true, + }; + // c4_base_seq sits just after C0..C3 = total_expected - 2. + assert_eq!(meta.c4_base_seq(), 4 + 2 + 1); +} + +#[test] +fn threshold_plan_assigns_canonical_seqs() { + let enc = |recipient: usize, row: usize, esi: usize| share_encryption_req(recipient, row, esi); + let plan = plan_threshold_dispatch( + pk_generation_req(), + share_computation_req(), + share_computation_req(), + vec![enc(2, 0, 0), enc(3, 0, 0)], + vec![enc(2, 0, 0)], + ); + let seqs: Vec = plan.iter().map(|i| i.seq).collect(); + assert_eq!(seqs, vec![1, 2, 3, 4, 5, 6]); + assert!(matches!(plan[0].kind, ThresholdProofKind::PkGeneration)); + assert!(matches!( + plan[3].kind, + ThresholdProofKind::SkShareEncryption { .. } + )); + assert!(matches!( + plan[5].kind, + ThresholdProofKind::ESmShareEncryption { .. } + )); +} + +#[test] +fn decryption_plan_assigns_offset_seqs() { + let plan = plan_decryption_dispatch( + dkg_share_decryption_req(), + vec![dkg_share_decryption_req(), dkg_share_decryption_req()], + 7, + ); + let seqs: Vec = plan.iter().map(|i| i.seq).collect(); + assert_eq!(seqs, vec![7, 8, 9]); + assert!(matches!(plan[0].kind, DecryptionProofKind::SecretKey)); + assert!(matches!( + plan[1].kind, + DecryptionProofKind::SmudgingNoise { esi_idx: 0 } + )); +} diff --git a/crates/zk-prover/src/proof_verification/actor.rs b/crates/zk-prover/src/proof_verification/actor.rs new file mode 100644 index 0000000000..9f5bbeac4d --- /dev/null +++ b/crates/zk-prover/src/proof_verification/actor.rs @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Verifies `EncryptionKeyReceived` events: recovers ECDSA address, delegates +//! ZK proof to `ZkActor`, and on failure emits [`SignedProofFailed`] for +//! on-chain fault attribution. + +use std::collections::HashMap; +use std::sync::Arc; + +use actix::{Actor, Addr, AsyncContext, Context, Handler, Message, Recipient}; +use alloy::primitives::{keccak256, Address, Bytes}; +use alloy::sol_types::SolValue; +use e3_events::{ + BusHandle, Committee, E3id, EncryptionKey, EncryptionKeyCreated, EncryptionKeyReceived, + EventContext, EventPublisher, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, + Proof, ProofType, ProofVerificationFailed, ProofVerificationPassed, Sequenced, + SignedProofFailed, SignedProofPayload, TypedEvent, +}; +use e3_fhe_params::BfvPreset; +use e3_request::E3Meta; +use e3_utils::NotifySync; +use e3_zk_helpers::{compute_dkg_pk_commitment_from_public_key_bytes, CiphernodesCommitteeSize}; +use tracing::{error, info, warn}; + +use crate::domain::proof_verification::{validate_external_key, validate_external_key_commitment}; + +#[derive(Debug, Message)] +#[rtype(result = "()")] +pub struct ZkVerificationRequest { + pub proof: Proof, + pub e3_id: E3id, + pub key: Arc, + pub sender: Recipient>, + pub artifacts_dir: String, +} + +#[derive(Debug, Clone, Message)] +#[rtype(result = "()")] +pub struct ZkVerificationResponse { + pub verified: bool, + pub error: Option, + pub e3_id: E3id, + pub key: Arc, +} + +#[derive(Clone, Debug)] +struct PendingVerification { + signed_payload: SignedProofPayload, + recovered_signer: Address, +} + +pub struct ProofVerificationActor { + bus: BusHandle, + verifier: Recipient>, + pending: HashMap<(E3id, u64), PendingVerification>, + /// Tracks preset + committee per E3 so we can derive `artifacts_dir` for proof verification. + presets: HashMap, + /// Canonical finalized committee in party-id order. A C0 signer must own the party slot whose + /// BFV key it advertises; recovering any valid ECDSA address is not sufficient. + committees: HashMap>, +} + +impl ProofVerificationActor { + pub fn new( + bus: &BusHandle, + verifier: Recipient>, + persisted_committees: HashMap, + persisted_e3_metadata: HashMap, + ) -> Self { + let mut actor = Self { + bus: bus.clone(), + verifier, + pending: HashMap::new(), + presets: HashMap::new(), + committees: HashMap::new(), + }; + for (e3_id, meta) in persisted_e3_metadata { + actor.store_preset( + e3_id, + meta.params_preset, + meta.threshold_m, + meta.threshold_n, + ); + } + for (e3_id, committee) in persisted_committees { + actor.store_committee(e3_id, committee.members()); + } + actor + } + + pub fn setup( + bus: &BusHandle, + verifier: Recipient>, + persisted_committees: HashMap, + persisted_e3_metadata: HashMap, + ) -> Addr { + let addr = Self::new(bus, verifier, persisted_committees, persisted_e3_metadata).start(); + bus.subscribe(EventType::CiphernodeSelected, addr.clone().into()); + bus.subscribe(EventType::CommitteeFinalized, addr.clone().into()); + bus.subscribe(EventType::EncryptionKeyReceived, addr.clone().into()); + bus.subscribe(EventType::E3RequestComplete, addr.clone().into()); + addr + } + + fn store_preset( + &mut self, + e3_id: E3id, + preset: BfvPreset, + threshold_m: usize, + threshold_n: usize, + ) { + match CiphernodesCommitteeSize::from_threshold(threshold_m, threshold_n) { + Ok(committee) => { + self.presets.insert(e3_id, (preset, committee)); + } + Err(error) => { + error!( + %e3_id, + threshold_m, + threshold_n, + %error, + "ProofVerificationActor: unrecognised committee; C0 keys will be rejected" + ); + } + } + } + + fn store_committee(&mut self, e3_id: E3id, members: &[String]) { + match members + .iter() + .map(|node| node.parse()) + .collect::, _>>() + { + Ok(committee) => { + self.committees.insert(e3_id, committee); + } + Err(error) => { + error!( + %e3_id, + %error, + "Finalized committee contains an invalid address; C0 keys will be rejected" + ); + } + } + } +} + +#[path = "effects.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +#[cfg(test)] +#[path = "actor_tests.rs"] +mod tests; diff --git a/crates/zk-prover/src/proof_verification/actor_tests.rs b/crates/zk-prover/src/proof_verification/actor_tests.rs new file mode 100644 index 0000000000..e57acd8c67 --- /dev/null +++ b/crates/zk-prover/src/proof_verification/actor_tests.rs @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use actix::{Actor, Context, Handler}; +use alloy::signers::local::PrivateKeySigner; +use e3_events::{ + hlc_factory::HlcFactory, EventBus, EventBusBarrier, EventBusConfig, EventPublisher, + ProofPayload, Seed, Sequencer, StoreEventRequested, StoreEventResponse, +}; +use e3_fhe_params::build_pair_for_preset; +use e3_utils::utility_types::ArcBytes; +use fhe::bfv::{PublicKey, SecretKey}; +use fhe_traits::Serialize; +use tokio::sync::mpsc; + +#[derive(Default)] +struct TestEventStore { + next_seq: u64, +} + +impl Actor for TestEventStore { + type Context = Context; +} + +impl Handler for TestEventStore { + type Result = (); + + fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) { + let StoreEventRequested { event, sender } = msg; + let seq = self.next_seq; + self.next_seq += 1; + sender.do_send(StoreEventResponse(event.into_sequenced(seq))); + } +} + +#[derive(Debug, PartialEq, Eq)] +struct ObservedVerification { + e3_id: E3id, + party_id: u64, + artifacts_dir: String, +} + +struct VerificationRecorder { + sender: mpsc::UnboundedSender, +} + +impl Actor for VerificationRecorder { + type Context = Context; +} + +impl Handler> for VerificationRecorder { + type Result = (); + + fn handle(&mut self, request: TypedEvent, _: &mut Self::Context) { + let (request, _) = request.into_components(); + let _ = self.sender.send(ObservedVerification { + e3_id: request.e3_id, + party_id: request.key.party_id, + artifacts_dir: request.artifacts_dir, + }); + } +} + +fn test_bus() -> BusHandle { + let event_bus = EventBus::::new(EventBusConfig { deduplicate: true }).start(); + let store = TestEventStore::default().start(); + let sequencer = Sequencer::new(&event_bus, store.recipient()).start(); + BusHandle::new(event_bus, sequencer, HlcFactory::new()).enable("c0-recovery-test") +} + +fn signed_c0_key( + signer: &PrivateKeySigner, + party_id: u64, + e3_id: &E3id, + preset: BfvPreset, +) -> Arc { + let (_, dkg_params) = build_pair_for_preset(preset).expect("build BFV test parameters"); + let mut rng = rand::rng(); + let secret_key = SecretKey::random(&dkg_params, &mut rng); + let public_key = PublicKey::new(&secret_key, &mut rng); + let public_key_bytes = public_key.to_bytes(); + let commitment = compute_dkg_pk_commitment_from_public_key_bytes(&public_key_bytes, preset) + .expect("compute C0 key commitment"); + let proof = Proof::new( + ProofType::C0PkBfv.circuit_names()[0], + ArcBytes::from_bytes(&[1, 2, 3]), + ArcBytes::from_bytes(&commitment), + ); + let signed = SignedProofPayload::sign( + ProofPayload { + e3_id: e3_id.clone(), + proof_type: ProofType::C0PkBfv, + proof: proof.clone(), + }, + signer, + ) + .expect("sign C0 test proof"); + + Arc::new( + EncryptionKey::new(party_id, ArcBytes::from_bytes(&public_key_bytes)) + .with_proof(proof) + .with_signed_payload(signed), + ) +} + +#[actix::test] +async fn restored_context_dispatches_c0_without_replayed_lifecycle_events() { + let bus = test_bus(); + let e3_id = E3id::new("7", 31_337); + let preset = BfvPreset::InsecureThreshold512; + let signer = PrivateKeySigner::random(); + let mut committee_members = vec![ + signer.address().to_string(), + PrivateKeySigner::random().address().to_string(), + PrivateKeySigner::random().address().to_string(), + ]; + committee_members.sort_by_key(|member| member.to_lowercase()); + let party_id = committee_members + .iter() + .position(|member| member.eq_ignore_ascii_case(&signer.address().to_string())) + .expect("signer belongs to restored committee") as u64; + + let persisted_committees = HashMap::from([(e3_id.clone(), Committee::new(committee_members))]); + let persisted_e3_metadata = HashMap::from([( + e3_id.clone(), + E3Meta { + threshold_m: 1, + threshold_n: 3, + seed: Seed([0; 32]), + params_preset: preset, + params: ArcBytes::default(), + error_size: ArcBytes::default(), + proof_aggregation_enabled: false, + }, + )]); + let (observed_tx, mut observed_rx) = mpsc::unbounded_channel(); + let verifier = VerificationRecorder { + sender: observed_tx, + } + .start(); + ProofVerificationActor::setup( + &bus, + verifier.recipient(), + persisted_committees, + persisted_e3_metadata, + ); + + // No CiphernodeSelected or CommitteeFinalized event is published: both are already + // represented by durable snapshots and may be outside the post-snapshot replay range. + bus.event_bus() + .send(EventBusBarrier) + .await + .expect("event bus barrier"); + bus.publish_without_context(EncryptionKeyReceived { + e3_id: e3_id.clone(), + key: signed_c0_key(&signer, party_id, &e3_id, preset), + }) + .expect("publish external C0 key"); + + let observed = tokio::time::timeout(std::time::Duration::from_secs(2), observed_rx.recv()) + .await + .expect("restored C0 context did not dispatch verification before timeout") + .expect("verification recorder stopped"); + assert_eq!( + observed, + ObservedVerification { + e3_id, + party_id, + artifacts_dir: preset + .artifacts_dir_for_committee(CiphernodesCommitteeSize::Minimum.as_str()), + } + ); +} diff --git a/crates/zk-prover/src/proof_verification/effects.rs b/crates/zk-prover/src/proof_verification/effects.rs new file mode 100644 index 0000000000..e953b4c40d --- /dev/null +++ b/crates/zk-prover/src/proof_verification/effects.rs @@ -0,0 +1,128 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! C0 signature, commitment, and proof-dispatch boundary. + +use super::*; + +impl ProofVerificationActor { + pub(in crate::actors::proof_verification) fn handle_encryption_key_received( + &mut self, + msg: TypedEvent, + ctx: &Context, + ) { + let (msg, ec) = msg.into_components(); + let pending_key = (msg.e3_id.clone(), msg.key.party_id); + if self.pending.contains_key(&pending_key) { + warn!( + e3_id = %msg.e3_id, + party_id = msg.key.party_id, + "C0 verification is already pending for party — ignoring duplicate" + ); + return; + } + + let Some((preset, committee_size)) = self.presets.get(&msg.e3_id).copied() else { + error!( + "No BfvPreset known for e3_id={} — cannot determine circuit artifacts directory. \ + This can happen if CiphernodeSelected was missed (e.g. after restart). Rejecting key from party {}.", + msg.e3_id, msg.key.party_id + ); + return; + }; + let Some(expected_signer) = self + .committees + .get(&msg.e3_id) + .and_then(|committee| { + usize::try_from(msg.key.party_id) + .ok() + .and_then(|i| committee.get(i)) + }) + .copied() + else { + error!( + e3_id = %msg.e3_id, + party_id = msg.key.party_id, + "No finalized committee member for C0 party slot — rejecting key" + ); + return; + }; + let validated = match validate_external_key( + &msg.e3_id, + &expected_signer, + msg.key.party_id, + msg.key.proof.as_ref(), + msg.key.signed_payload.as_ref(), + ) { + Ok(validated) => validated, + Err(reason) => { + error!("{reason}"); + return; + } + }; + let proof = msg + .key + .proof + .clone() + .expect("proof present after validation"); + let key_commitment = + match compute_dkg_pk_commitment_from_public_key_bytes(&msg.key.pk_bfv, preset) { + Ok(commitment) => commitment, + Err(err) => { + error!( + e3_id = %msg.e3_id, + party_id = msg.key.party_id, + "Could not bind C0 proof to advertised BFV key: {err} — rejecting key" + ); + return; + } + }; + if let Err(reason) = + validate_external_key_commitment(msg.key.party_id, &proof, &key_commitment) + { + error!("{reason}"); + return; + } + + // Store the signed payload so we can reference it in the verification response + self.pending.insert( + pending_key, + PendingVerification { + signed_payload: validated.signed_payload, + recovered_signer: validated.recovered_signer, + }, + ); + + let artifacts_dir = preset.artifacts_dir_for_committee(committee_size.as_str()); + + let request = TypedEvent::new( + ZkVerificationRequest { + proof: proof.clone(), + e3_id: msg.e3_id, + key: msg.key, + sender: ctx.address().recipient(), + artifacts_dir, + }, + ec, + ); + + self.verifier.do_send(request); + } + + pub(in crate::actors::proof_verification) fn publish_key_created( + &self, + e3_id: E3id, + key: Arc, + ec: EventContext, + ) { + if let Err(err) = self.bus.publish( + EncryptionKeyCreated { + e3_id, + key, + external: true, + }, + ec, + ) { + error!("Failed to publish EncryptionKeyCreated: {err}"); + } + } +} diff --git a/crates/zk-prover/src/proof_verification/handlers.rs b/crates/zk-prover/src/proof_verification/handlers.rs new file mode 100644 index 0000000000..a95b4eba49 --- /dev/null +++ b/crates/zk-prover/src/proof_verification/handlers.rs @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Event routing and verification-result publication. + +use super::*; + +impl Actor for ProofVerificationActor { + type Context = Context; +} + +impl Handler for ProofVerificationActor { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + match msg { + InterfoldEventData::CiphernodeSelected(data) => { + self.store_preset( + data.e3_id, + data.params_preset, + data.threshold_m, + data.threshold_n, + ); + } + InterfoldEventData::CommitteeFinalized(mut data) => { + // The EVM decoder already emits canonical address order, but sorting again keeps + // this trust boundary correct for replayed/test-produced events as well. + data.sort_by_score(); + self.store_committee(data.e3_id, &data.committee); + } + InterfoldEventData::EncryptionKeyReceived(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::E3RequestComplete(data) => { + let e3_id = data.e3_id; + self.presets.remove(&e3_id); + self.committees.remove(&e3_id); + self.pending + .retain(|(pending_e3, _), _| pending_e3 != &e3_id); + } + _ => (), + } + } +} + +impl Handler> for ProofVerificationActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_encryption_key_received(msg, ctx) + } +} + +impl Handler> for ProofVerificationActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + let (msg, ec) = msg.into_components(); + let pending_key = (msg.e3_id.clone(), msg.key.party_id); + let pending = self.pending.remove(&pending_key); + + if msg.verified { + let Some(PendingVerification { + signed_payload, + recovered_signer, + }) = pending + else { + warn!( + "No pending verification for verified party {} — ignoring duplicate response", + msg.key.party_id + ); + return; + }; + + info!( + "C0 proof verified for party {} - accepting key", + msg.key.party_id + ); + let party_id = msg.key.party_id; + let e3_id = msg.e3_id.clone(); + self.publish_key_created(msg.e3_id, msg.key, ec.clone()); + + // Emit ProofVerificationPassed so AccusationManager can cache success + { + let data_hash: [u8; 32] = { + let msg = ( + Bytes::copy_from_slice(&signed_payload.payload.proof.data), + Bytes::copy_from_slice(&signed_payload.payload.proof.public_signals), + ) + .abi_encode(); + keccak256(&msg).into() + }; + if let Err(err) = self.bus.publish( + ProofVerificationPassed { + e3_id, + party_id, + address: recovered_signer, + proof_type: ProofType::C0PkBfv, + data_hash, + public_signals: signed_payload.payload.proof.public_signals.clone(), + proof_data: signed_payload.payload.proof.data.clone(), + }, + ec, + ) { + error!("Failed to publish ProofVerificationPassed: {err}"); + } + } + } else { + let error_msg = msg.error.unwrap_or_else(|| "unknown error".to_string()); + error!( + "C0 proof verification FAILED for party {} - rejecting key and stopping E3: {}", + msg.key.party_id, error_msg + ); + + if let Some(PendingVerification { + signed_payload, + recovered_signer, + }) = pending + { + warn!( + "Emitting SignedProofFailed for party {} (address: {recovered_signer})", + msg.key.party_id + ); + if let Err(err) = self.bus.publish( + SignedProofFailed { + e3_id: msg.e3_id.clone(), + faulting_node: recovered_signer, + proof_type: signed_payload.payload.proof_type, + signed_payload: signed_payload.clone(), + }, + ec.clone(), + ) { + error!("Failed to publish SignedProofFailed: {err}"); + } + + // Emit ProofVerificationFailed for AccusationManager + let data_hash: [u8; 32] = { + let msg = ( + Bytes::copy_from_slice(&signed_payload.payload.proof.data), + Bytes::copy_from_slice(&signed_payload.payload.proof.public_signals), + ) + .abi_encode(); + keccak256(&msg).into() + }; + if let Err(err) = self.bus.publish( + ProofVerificationFailed { + e3_id: msg.e3_id.clone(), + accused_party_id: msg.key.party_id, + accused_address: recovered_signer, + proof_type: ProofType::C0PkBfv, + data_hash, + signed_payload, + }, + ec.clone(), + ) { + error!("Failed to publish ProofVerificationFailed: {err}"); + } + } + + // NOTE: We do NOT emit E3Failed here. The on-chain SlashingManager + // will expel the faulting node and check if the committee drops below + // threshold. If it does, the contract emits E3Failed on-chain, which + // the EVM reader picks up and propagates to all actors. If the committee + // is still above threshold, the DKG continues with N-1 nodes. + } + } +} diff --git a/crates/zk-prover/src/domain/proof_verification.rs b/crates/zk-prover/src/proof_verification/workflow.rs similarity index 100% rename from crates/zk-prover/src/domain/proof_verification.rs rename to crates/zk-prover/src/proof_verification/workflow.rs diff --git a/crates/zk-prover/src/share_verification/actor.rs b/crates/zk-prover/src/share_verification/actor.rs new file mode 100644 index 0000000000..46beaaaf53 --- /dev/null +++ b/crates/zk-prover/src/share_verification/actor.rs @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Actor for C2/C3/C4 share proof verification. +//! +//! Follows the same pattern as [`ProofVerificationActor`] (for C0) — sits +//! between the raw proof data and the verified result, handling ECDSA validation +//! and ZK verification orchestration. +//! +//! ## Flow +//! +//! 1. Receives [`ShareVerificationDispatched`] from [`ThresholdKeyshare`]. +//! 2. Performs ECDSA validation (signature recovery, signer consistency, e3_id, +//! circuit name) — lightweight, no thread pool needed. +//! 3. Dispatches ZK-only verification to multithread via [`ComputeRequest`]. +//! 4. Receives [`ComputeResponse`] from multithread with pure ZK results. +//! 5. Combines ECDSA + ZK results. +//! 6. Emits [`SignedProofFailed`] for any failing proofs. +//! 7. Publishes [`ShareVerificationComplete`] with dishonest party set. + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use actix::{Actor, Addr, Context, Handler}; +use alloy::primitives::{keccak256, Address, Bytes}; +use alloy::sol_types::SolValue; +use e3_events::{ + BusHandle, CommitmentConsistencyCheckComplete, CommitmentConsistencyCheckRequested, Committee, + ComputeRequest, ComputeRequestError, ComputeResponse, ComputeResponseKind, CorrelationId, E3id, + EventContext, EventPublisher, EventSubscriber, EventType, InterfoldEvent, InterfoldEventData, + PartyVerificationResult, ProofType, ProofVerificationFailed, ProofVerificationPassed, + Sequenced, ShareVerificationComplete, ShareVerificationDispatched, SignedProofFailed, + SignedProofPayload, TypedEvent, VerificationKind, VerifyShareDecryptionProofsRequest, + VerifyShareProofsRequest, ZkRequest, ZkResponse, +}; +use e3_utils::utility_types::ArcBytes; +use e3_utils::NotifySync; +use tracing::{error, info, warn}; + +use crate::workflow::share_verification::{ + filter_consistent, label_for, PendingConsistencyCheck, PendingVerification, ShareVerifier, + VerifiableParty, ZkPartyEmission, +}; + +/// Actor that handles C1/C2/C3/C4/C6 share proof verification. +/// +/// Three-stage pipeline: +/// 1. ECDSA validation (lightweight, done inline) +/// 2. Commitment consistency check (dispatched to per-E3 checker via event bus) +/// 3. ZK proof verification (heavyweight, delegated to multithread) +/// +/// Emits [`SignedProofFailed`] for fault attribution and +/// [`ShareVerificationComplete`] with the final dishonest party set. +pub struct ShareVerificationActor { + bus: BusHandle, + /// Canonical finalized committees in party-id order. Every signed C1-C4/C6 proof must recover + /// to the address that owns its outer `sender_party_id` slot. + committees: HashMap>, + /// Tracks pending ZK verifications by correlation ID. + pending: HashMap, + /// Tracks pending consistency checks by correlation ID (between ECDSA and ZK). + pending_consistency: HashMap, +} + +impl ShareVerificationActor { + pub fn new(bus: &BusHandle, persisted_committees: HashMap) -> Self { + let mut actor = Self { + bus: bus.clone(), + committees: HashMap::new(), + pending: HashMap::new(), + pending_consistency: HashMap::new(), + }; + for (e3_id, committee) in persisted_committees { + actor.store_committee(e3_id, committee.members()); + } + actor + } + + pub fn setup(bus: &BusHandle, persisted_committees: HashMap) -> Addr { + let addr = Self::new(bus, persisted_committees).start(); + bus.subscribe(EventType::CommitteeFinalized, addr.clone().into()); + bus.subscribe(EventType::ShareVerificationDispatched, addr.clone().into()); + bus.subscribe(EventType::ComputeResponse, addr.clone().into()); + bus.subscribe(EventType::ComputeRequestError, addr.clone().into()); + bus.subscribe( + EventType::CommitmentConsistencyCheckComplete, + addr.clone().into(), + ); + bus.subscribe(EventType::E3RequestComplete, addr.clone().into()); + addr + } + + fn store_committee(&mut self, e3_id: E3id, members: &[String]) { + match members + .iter() + .map(|member| member.parse()) + .collect::, _>>() + { + Ok(committee) => { + self.committees.insert(e3_id, committee); + } + Err(error) => { + error!( + %e3_id, + %error, + "Finalized committee contains an invalid address; share proofs will be rejected" + ); + } + } + } +} + +#[path = "effects/mod.rs"] +mod effects; +#[path = "handlers.rs"] +mod handlers; + +#[cfg(test)] +#[path = "actor_tests.rs"] +mod tests; diff --git a/crates/zk-prover/src/share_verification/actor_tests.rs b/crates/zk-prover/src/share_verification/actor_tests.rs new file mode 100644 index 0000000000..486586890c --- /dev/null +++ b/crates/zk-prover/src/share_verification/actor_tests.rs @@ -0,0 +1,117 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use actix::{Actor, Context, Handler}; +use alloy::signers::local::PrivateKeySigner; +use e3_events::{ + hlc_factory::HlcFactory, Event, EventBus, EventBusBarrier, EventBusConfig, EventPublisher, + Proof, ProofPayload, Sequencer, StoreEventRequested, StoreEventResponse, +}; +use e3_fhe_params::BfvPreset; +use e3_utils::utility_types::ArcBytes; +use e3_zk_helpers::CiphernodesCommitteeSize; +use std::{collections::BTreeSet, time::Duration}; + +#[derive(Default)] +struct TestEventStore { + next_seq: u64, +} + +impl Actor for TestEventStore { + type Context = Context; +} + +impl Handler for TestEventStore { + type Result = (); + + fn handle(&mut self, msg: StoreEventRequested, _: &mut Self::Context) { + let StoreEventRequested { event, sender } = msg; + let seq = self.next_seq; + self.next_seq += 1; + sender.do_send(StoreEventResponse(event.into_sequenced(seq))); + } +} + +fn test_bus() -> BusHandle { + let event_bus = EventBus::::new(EventBusConfig { deduplicate: true }).start(); + let store = TestEventStore::default().start(); + let sequencer = Sequencer::new(&event_bus, store.recipient()).start(); + BusHandle::new(event_bus, sequencer, HlcFactory::new()).enable("share-recovery-test") +} + +fn signed_c6(signer: &PrivateKeySigner, e3_id: &E3id, marker: u8) -> SignedProofPayload { + let proof_type = ProofType::C6ThresholdShareDecryption; + let proof = Proof::new( + proof_type.circuit_names()[0], + ArcBytes::from_bytes(&[marker, 2, 3]), + ArcBytes::from_bytes(&[4, 5, marker]), + ); + SignedProofPayload::sign( + ProofPayload { + e3_id: e3_id.clone(), + proof_type, + proof, + }, + signer, + ) + .expect("sign C6 test proof") +} + +#[actix::test] +async fn restored_committee_authorizes_c6_without_replayed_finalization_event() { + let bus = test_bus(); + let e3_id = E3id::new("0", 31_337); + let signers = [ + PrivateKeySigner::random(), + PrivateKeySigner::random(), + PrivateKeySigner::random(), + ]; + let committee = Committee::new( + signers + .iter() + .map(|signer| signer.address().to_string()) + .collect(), + ); + ShareVerificationActor::setup(&bus, HashMap::from([(e3_id.clone(), committee)])); + + // Register the observer and fence the EventBus before dispatching. No + // CommitteeFinalized event is published: this models restart after that event was + // incorporated into the durable snapshot and excluded from the replay range. + let consistency_requested = bus.wait_for(EventType::CommitmentConsistencyCheckRequested); + bus.event_bus() + .send(EventBusBarrier) + .await + .expect("event bus barrier"); + + bus.publish_without_context(ShareVerificationDispatched { + e3_id: e3_id.clone(), + kind: VerificationKind::ThresholdDecryptionProofs, + share_proofs: signers[..2] + .iter() + .enumerate() + .map(|(party_id, signer)| e3_events::PartyProofsToVerify { + sender_party_id: party_id as u64, + signed_proofs: vec![signed_c6(signer, &e3_id, party_id as u8)], + }) + .collect(), + decryption_proofs: Vec::new(), + pre_dishonest: BTreeSet::new(), + params_preset: BfvPreset::InsecureThreshold512, + committee_size: CiphernodesCommitteeSize::Minimum, + }) + .expect("publish C6 verification dispatch"); + + let event = tokio::time::timeout(Duration::from_secs(2), consistency_requested) + .await + .expect("restored committee did not authorize C6 before timeout") + .expect("wait for consistency request"); + let InterfoldEventData::CommitmentConsistencyCheckRequested(request) = event.into_data() else { + panic!("unexpected event type") + }; + assert_eq!(request.e3_id, e3_id); + assert_eq!(request.party_proofs.len(), 2); +} diff --git a/crates/zk-prover/src/share_verification/effects/complete_verification.rs b/crates/zk-prover/src/share_verification/effects/complete_verification.rs new file mode 100644 index 0000000000..96ae78f3ce --- /dev/null +++ b/crates/zk-prover/src/share_verification/effects/complete_verification.rs @@ -0,0 +1,26 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Publish the terminal verification decision. + +use super::*; + +impl ShareVerificationActor { + pub(in crate::actors::share_verification) fn publish_complete( + &self, + e3_id: E3id, + kind: VerificationKind, + dishonest_parties: BTreeSet, + ec: EventContext, + ) { + if let Err(err) = self.bus.publish( + ShareVerificationComplete { + e3_id, + kind, + dishonest_parties, + }, + ec, + ) { + error!("Failed to publish ShareVerificationComplete: {err}"); + } + } +} diff --git a/crates/zk-prover/src/share_verification/effects/consistency.rs b/crates/zk-prover/src/share_verification/effects/consistency.rs new file mode 100644 index 0000000000..d2926aa11c --- /dev/null +++ b/crates/zk-prover/src/share_verification/effects/consistency.rs @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Apply commitment-consistency results and dispatch heavy ZK verification. + +use super::*; + +impl ShareVerificationActor { + /// Handle consistency check response: add inconsistent parties to the + /// dishonest set, then dispatch ZK verification for the remaining + /// consistent parties. + pub(in crate::actors::share_verification) fn handle_consistency_check_complete( + &mut self, + msg: TypedEvent, + ) { + let (data, _ec) = msg.into_components(); + + let Some(pending) = self.pending_consistency.remove(&data.correlation_id) else { + return; // Not our correlation ID + }; + + let label = label_for(&pending.kind); + + if !data.inconsistent_parties.is_empty() { + warn!( + "{} consistency check found {} inconsistent parties for E3 {}: {:?}", + label, + data.inconsistent_parties.len(), + pending.e3_id, + data.inconsistent_parties + ); + } + + // Accumulate all dishonest parties discovered so far + let mut dishonest_so_far: BTreeSet = pending.pre_dishonest.clone(); + dishonest_so_far.extend(&pending.ecdsa_dishonest); + dishonest_so_far.extend(&data.inconsistent_parties); + + // Filter ECDSA-passed proofs to only consistent parties and dispatch ZK + let inconsistent = &data.inconsistent_parties; + let zk_correlation_id = CorrelationId::new(); + + let (request, dispatched_party_ids) = match pending.kind { + VerificationKind::ShareProofs + | VerificationKind::ThresholdDecryptionProofs + | VerificationKind::PkGenerationProofs => { + let Some((passed, ids)) = + filter_consistent(pending.ecdsa_passed_share_proofs, inconsistent, |p| { + p.sender_party_id + }) + else { + self.publish_complete( + pending.e3_id, + pending.kind, + dishonest_so_far, + pending.ec, + ); + return; + }; + let req = ComputeRequest::zk( + ZkRequest::VerifyShareProofs(VerifyShareProofsRequest { + party_proofs: passed, + params_preset: pending.params_preset, + committee_size: pending.committee_size, + }), + zk_correlation_id, + pending.e3_id.clone(), + ); + (req, ids) + } + VerificationKind::DecryptionProofs => { + let Some((passed, ids)) = + filter_consistent(pending.ecdsa_passed_decryption_proofs, inconsistent, |p| { + p.sender_party_id + }) + else { + self.publish_complete( + pending.e3_id, + pending.kind, + dishonest_so_far, + pending.ec, + ); + return; + }; + let req = ComputeRequest::zk( + ZkRequest::VerifyShareDecryptionProofs(VerifyShareDecryptionProofsRequest { + party_proofs: passed, + params_preset: pending.params_preset, + committee_size: pending.committee_size, + }), + zk_correlation_id, + pending.e3_id.clone(), + ); + (req, ids) + } + }; + + // Only keep proof hashes/signals/addresses for parties going to ZK + let party_addresses: HashMap = pending + .party_addresses + .into_iter() + .filter(|(pid, _)| dispatched_party_ids.contains(pid)) + .collect(); + let party_proof_hashes: HashMap> = pending + .party_proof_hashes + .into_iter() + .filter(|(pid, _)| dispatched_party_ids.contains(pid)) + .collect(); + let party_public_signals: HashMap> = pending + .party_public_signals + .into_iter() + .filter(|(pid, _)| dispatched_party_ids.contains(pid)) + .collect(); + let party_proof_data: HashMap> = pending + .party_proof_data + .into_iter() + .filter(|(pid, _)| dispatched_party_ids.contains(pid)) + .collect(); + + // Store pending ZK verification state. + // All prior dishonest parties (pre_dishonest + ECDSA + consistency) are + // folded into `pre_dishonest` so that `handle_compute_response` produces + // the correct final dishonest set when it adds ZK failures. + self.pending.insert( + zk_correlation_id, + PendingVerification { + e3_id: pending.e3_id.clone(), + kind: pending.kind.clone(), + ec: pending.ec.clone(), + ecdsa_dishonest: HashSet::new(), + pre_dishonest: dishonest_so_far, + dispatched_party_ids: dispatched_party_ids.clone(), + party_addresses, + party_proof_hashes, + party_public_signals, + party_proof_data, + params_preset: pending.params_preset, + committee_size: pending.committee_size, + }, + ); + + if let Err(err) = self.bus.publish(request, pending.ec.clone()) { + error!( + "Failed to dispatch {} ZK verification after consistency check: {err}", + label + ); + if let Some(zk_pending) = self.pending.remove(&zk_correlation_id) { + let mut all_dishonest: BTreeSet = zk_pending.pre_dishonest; + all_dishonest.extend(zk_pending.ecdsa_dishonest); + all_dishonest.extend(zk_pending.dispatched_party_ids); + self.publish_complete( + zk_pending.e3_id, + zk_pending.kind, + all_dishonest, + zk_pending.ec, + ); + } + } + } +} diff --git a/crates/zk-prover/src/share_verification/effects/dispatch.rs b/crates/zk-prover/src/share_verification/effects/dispatch.rs new file mode 100644 index 0000000000..f1afedc441 --- /dev/null +++ b/crates/zk-prover/src/share_verification/effects/dispatch.rs @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Select the verification workflow for an incoming proof bundle. + +use super::*; + +impl ShareVerificationActor { + pub(in crate::actors::share_verification) fn handle_share_verification_dispatched( + &mut self, + msg: TypedEvent, + ) { + let (msg, ec) = msg.into_components(); + let e3_id = msg.e3_id.clone(); + + info!( + "handling ShareVerificationDispatched {:?}, {:?}", + e3_id, msg.kind + ); + + let params_preset = msg.params_preset; + let committee_size = msg.committee_size; + match msg.kind { + VerificationKind::ShareProofs + | VerificationKind::ThresholdDecryptionProofs + | VerificationKind::PkGenerationProofs => { + let kind = msg.kind.clone(); + self.verify_proofs( + e3_id, + kind.clone(), + msg.share_proofs, + msg.pre_dishonest, + ec, + params_preset, + committee_size, + |pending, passed| { + pending.ecdsa_passed_share_proofs = passed; + }, + ); + } + VerificationKind::DecryptionProofs => { + self.verify_proofs( + e3_id, + VerificationKind::DecryptionProofs, + msg.decryption_proofs, + msg.pre_dishonest, + ec, + params_preset, + committee_size, + |pending, passed| { + pending.ecdsa_passed_decryption_proofs = passed; + }, + ); + } + } + } +} diff --git a/crates/zk-prover/src/share_verification/effects/ecdsa.rs b/crates/zk-prover/src/share_verification/effects/ecdsa.rs new file mode 100644 index 0000000000..a979ae478a --- /dev/null +++ b/crates/zk-prover/src/share_verification/effects/ecdsa.rs @@ -0,0 +1,107 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Validate proof ownership/signatures and dispatch consistency checking. + +use super::*; + +impl ShareVerificationActor { + /// Generic ECDSA validation + consistency check dispatch. + /// + /// After ECDSA validation, publishes [`CommitmentConsistencyCheckRequested`] + /// and stores a [`PendingConsistencyCheck`]. ZK verification is deferred + /// until the consistency check response arrives. + #[allow(clippy::too_many_arguments)] + pub(in crate::actors::share_verification) fn verify_proofs( + &mut self, + e3_id: E3id, + kind: VerificationKind, + party_proofs: Vec

, + pre_dishonest: BTreeSet, + ec: EventContext, + params_preset: e3_fhe_params::BfvPreset, + committee_size: e3_zk_helpers::CiphernodesCommitteeSize, + store_passed_proofs: impl FnOnce(&mut PendingConsistencyCheck, Vec

), + ) { + let e3_id_str = e3_id.to_string(); + let label = label_for(&kind); + + // Pure ECDSA validation + proof-commitment preparation lives in the + // domain service; the actor only emits failures, stores pending state, + // and publishes the consistency-check request. + let committee = self.committees.get(&e3_id).map(Vec::as_slice); + let outcome = ShareVerifier::validate_and_prepare( + &party_proofs, + &e3_id_str, + &kind, + label, + committee, + params_preset, + committee_size, + ); + + for failure in &outcome.failures { + self.emit_signed_proof_failed( + &e3_id, + &failure.signed, + failure.recovered, + failure.party_id, + &ec, + ); + } + + if outcome.ecdsa_passed_parties.is_empty() { + // All parties failed ECDSA — publish result immediately + let mut all_dishonest: BTreeSet = pre_dishonest; + all_dishonest.extend(outcome.ecdsa_dishonest); + self.publish_complete(e3_id, kind, all_dishonest, ec); + return; + } + + // Store pending consistency check with the original party proofs + let correlation_id = CorrelationId::new(); + let mut pending = PendingConsistencyCheck { + e3_id: e3_id.clone(), + kind: kind.clone(), + ec: ec.clone(), + ecdsa_dishonest: outcome.ecdsa_dishonest, + pre_dishonest, + party_addresses: outcome.party_addresses, + party_proof_hashes: outcome.party_proof_hashes, + party_public_signals: outcome.party_public_signals, + party_proof_data: outcome.party_proof_data, + ecdsa_passed_share_proofs: Vec::new(), + ecdsa_passed_decryption_proofs: Vec::new(), + params_preset, + committee_size, + }; + store_passed_proofs(&mut pending, outcome.ecdsa_passed_parties); + self.pending_consistency.insert(correlation_id, pending); + + // Publish consistency check request + if let Err(err) = self.bus.publish( + CommitmentConsistencyCheckRequested { + e3_id: e3_id.clone(), + kind: kind.clone(), + correlation_id, + party_proofs: outcome.consistency_party_data, + }, + ec.clone(), + ) { + error!( + "Failed to dispatch {} consistency check: {err} — treating all as dishonest", + label + ); + if let Some(pending) = self.pending_consistency.remove(&correlation_id) { + let mut all_dishonest: BTreeSet = pending.pre_dishonest; + all_dishonest.extend(pending.ecdsa_dishonest); + for p in &pending.ecdsa_passed_share_proofs { + all_dishonest.insert(p.sender_party_id); + } + for p in &pending.ecdsa_passed_decryption_proofs { + all_dishonest.insert(p.sender_party_id); + } + self.publish_complete(e3_id, kind, all_dishonest, ec); + } + } + } +} diff --git a/crates/zk-prover/src/share_verification/effects/mod.rs b/crates/zk-prover/src/share_verification/effects/mod.rs new file mode 100644 index 0000000000..b32a585d99 --- /dev/null +++ b/crates/zk-prover/src/share_verification/effects/mod.rs @@ -0,0 +1,15 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Share-verification effect execution grouped by business action. + +use super::*; + +mod complete_verification; +mod consistency; +mod dispatch; +mod ecdsa; +mod results; diff --git a/crates/zk-prover/src/share_verification/effects/results.rs b/crates/zk-prover/src/share_verification/effects/results.rs new file mode 100644 index 0000000000..a989af31bd --- /dev/null +++ b/crates/zk-prover/src/share_verification/effects/results.rs @@ -0,0 +1,185 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Tally ZK results, emit fault attribution, and handle worker errors. + +use super::*; + +impl ShareVerificationActor { + /// Handle ZK verification response from multithread. + pub(in crate::actors::share_verification) fn handle_compute_response( + &mut self, + msg: TypedEvent, + ) { + let (msg, _ec) = msg.into_components(); + + let correlation_id = msg.correlation_id; + let Some(pending) = self.pending.remove(&correlation_id) else { + return; // Not our correlation ID + }; + + let zk_results: Vec = match (&pending.kind, msg.response) { + ( + VerificationKind::ShareProofs + | VerificationKind::ThresholdDecryptionProofs + | VerificationKind::PkGenerationProofs, + ComputeResponseKind::Zk(ZkResponse::VerifyShareProofs(r)), + ) => r.party_results, + ( + VerificationKind::DecryptionProofs, + ComputeResponseKind::Zk(ZkResponse::VerifyShareDecryptionProofs(r)), + ) => r.party_results, + _ => { + error!("Unexpected ComputeResponse kind for verification — treating all dispatched parties as dishonest"); + let mut all_dishonest: BTreeSet = pending.pre_dishonest; + all_dishonest.extend(pending.ecdsa_dishonest); + all_dishonest.extend(pending.dispatched_party_ids); + self.publish_complete(pending.e3_id, pending.kind, all_dishonest, pending.ec); + return; + } + }; + + // Pure tally (dishonest accounting + emission decisions) lives in the + // domain service; the actor performs the resulting bus publishes. + let tally = ShareVerifier::tally_zk_results( + pending.pre_dishonest, + &pending.ecdsa_dishonest, + &pending.dispatched_party_ids, + &zk_results, + ); + + for emission in tally.emissions { + match emission { + ZkPartyEmission::Failed { party_id, signed } => { + let addr = pending.party_addresses.get(&party_id).copied(); + self.emit_signed_proof_failed( + &pending.e3_id, + &signed, + addr, + party_id, + &pending.ec, + ); + } + ZkPartyEmission::Passed { party_id } => { + // Emit ProofVerificationPassed for each proof type from this party + if let Some(hashes) = pending.party_proof_hashes.get(&party_id) { + let addr = pending + .party_addresses + .get(&party_id) + .copied() + .unwrap_or_default(); + let signals = pending.party_public_signals.get(&party_id); + let datas = pending.party_proof_data.get(&party_id); + for (i, &(proof_type, data_hash)) in hashes.iter().enumerate() { + let public_signals = signals + .and_then(|s| s.get(i)) + .map(|(_, ps)| ps.clone()) + .unwrap_or_default(); + let proof_data = datas + .and_then(|d| d.get(i)) + .map(|(_, pd)| pd.clone()) + .unwrap_or_default(); + if let Err(err) = self.bus.publish( + ProofVerificationPassed { + e3_id: pending.e3_id.clone(), + party_id, + address: addr, + proof_type, + data_hash, + public_signals, + proof_data, + }, + pending.ec.clone(), + ) { + error!("Failed to publish ProofVerificationPassed: {err}"); + } + } + } + } + } + } + + self.publish_complete(pending.e3_id, pending.kind, tally.dishonest, pending.ec); + } + + pub(in crate::actors::share_verification) fn emit_signed_proof_failed( + &self, + e3_id: &E3id, + signed_payload: &SignedProofPayload, + recovered_addr: Option

, + party_id: u64, + ec: &EventContext, + ) { + let faulting_node = match recovered_addr { + Some(addr) => addr, + None => match signed_payload.recover_address() { + Ok(addr) => addr, + Err(err) => { + warn!( + "Signature recovery failed for party {} — using zero address for fault attribution: {err}", + party_id + ); + Address::ZERO + } + }, + }; + + if let Err(err) = self.bus.publish( + SignedProofFailed { + e3_id: e3_id.clone(), + faulting_node, + proof_type: signed_payload.payload.proof_type, + signed_payload: signed_payload.clone(), + }, + ec.clone(), + ) { + error!("Failed to publish SignedProofFailed: {err}"); + } + + // Also emit ProofVerificationFailed for AccusationManager + let data_hash: [u8; 32] = { + let msg = ( + Bytes::copy_from_slice(&signed_payload.payload.proof.data), + Bytes::copy_from_slice(&signed_payload.payload.proof.public_signals), + ) + .abi_encode(); + keccak256(&msg).into() + }; + if let Err(err) = self.bus.publish( + ProofVerificationFailed { + e3_id: e3_id.clone(), + accused_party_id: party_id, + accused_address: faulting_node, + proof_type: signed_payload.payload.proof_type, + data_hash, + signed_payload: signed_payload.clone(), + }, + ec.clone(), + ) { + error!("Failed to publish ProofVerificationFailed: {err}"); + } + } + + /// Handle computation error from multithread — clean up pending state and + /// publish ShareVerificationComplete treating all dispatched parties as dishonest. + pub(in crate::actors::share_verification) fn handle_compute_request_error( + &mut self, + msg: TypedEvent, + ) { + let (msg, _ec) = msg.into_components(); + + let correlation_id = msg.correlation_id(); + let Some(pending) = self.pending.remove(correlation_id) else { + return; + }; + + error!( + "ZK verification computation failed for E3 {} ({:?}): {} — treating all dispatched parties as dishonest", + pending.e3_id, pending.kind, msg + ); + + let mut all_dishonest: BTreeSet = pending.pre_dishonest; + all_dishonest.extend(pending.ecdsa_dishonest); + all_dishonest.extend(pending.dispatched_party_ids); + self.publish_complete(pending.e3_id, pending.kind, all_dishonest, pending.ec); + } +} diff --git a/crates/zk-prover/src/share_verification/handlers.rs b/crates/zk-prover/src/share_verification/handlers.rs new file mode 100644 index 0000000000..096c868b22 --- /dev/null +++ b/crates/zk-prover/src/share_verification/handlers.rs @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Actix envelope routing for proof verification. + +use super::*; + +impl Actor for ShareVerificationActor { + type Context = Context; +} + +impl Handler for ShareVerificationActor { + type Result = (); + + fn handle(&mut self, msg: InterfoldEvent, ctx: &mut Self::Context) -> Self::Result { + let (msg, ec) = msg.into_components(); + match msg { + InterfoldEventData::CommitteeFinalized(mut data) => { + // Mirror the C0 verifier's canonical ordering at this trust boundary. Replayed and + // test-produced events are not assumed to have passed through the EVM decoder. + data.sort_by_score(); + self.store_committee(data.e3_id, &data.committee); + } + InterfoldEventData::ShareVerificationDispatched(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ComputeResponse(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::ComputeRequestError(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::CommitmentConsistencyCheckComplete(data) => { + self.notify_sync(ctx, TypedEvent::new(data, ec)) + } + InterfoldEventData::E3RequestComplete(data) => { + let e3_id = data.e3_id; + self.committees.remove(&e3_id); + self.pending.retain(|_, pending| pending.e3_id != e3_id); + self.pending_consistency + .retain(|_, pending| pending.e3_id != e3_id); + } + _ => (), + } + } +} + +impl Handler> for ShareVerificationActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_share_verification_dispatched(msg) + } +} + +impl Handler> for ShareVerificationActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_compute_response(msg) + } +} + +impl Handler> for ShareVerificationActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_compute_request_error(msg) + } +} + +impl Handler> for ShareVerificationActor { + type Result = (); + + fn handle( + &mut self, + msg: TypedEvent, + _ctx: &mut Self::Context, + ) -> Self::Result { + self.handle_consistency_check_complete(msg) + } +} diff --git a/crates/zk-prover/src/share_verification/state.rs b/crates/zk-prover/src/share_verification/state.rs new file mode 100644 index 0000000000..3dade869eb --- /dev/null +++ b/crates/zk-prover/src/share_verification/state.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Verification inputs, pending state, and pure decision types. + +use super::*; + +/// Trait for party types whose signed proofs can be ECDSA-validated and ZK-verified. +pub(crate) trait VerifiableParty: Clone + PartialEq { + fn party_id(&self) -> u64; + fn signed_proofs(&self) -> Vec; +} + +impl VerifiableParty for PartyProofsToVerify { + fn party_id(&self) -> u64 { + self.sender_party_id + } + fn signed_proofs(&self) -> Vec { + self.signed_proofs.clone() + } +} + +impl VerifiableParty for PartyShareDecryptionProofsToVerify { + fn party_id(&self) -> u64 { + self.sender_party_id + } + fn signed_proofs(&self) -> Vec { + std::iter::once(self.signed_sk_decryption_proof.clone()) + .chain(self.signed_e_sm_decryption_proofs.iter().cloned()) + .collect() + } +} + +/// ECDSA validation result for a single party. +pub(crate) struct EcdsaPartyResult { + pub(crate) passed: bool, + /// The pair (signed_payload, recovered_address) of the first failing proof, if any. + pub(crate) failed_payload: Option<(SignedProofPayload, Option
)>, +} + +/// A single ECDSA failure to be attributed (emitted) by the actor. +pub(crate) struct EcdsaFailure { + pub(crate) party_id: u64, + pub(crate) signed: SignedProofPayload, + pub(crate) recovered: Option
, +} + +/// Outcome of validating + preparing a batch of party proofs for the +/// consistency-check + ZK phases. Pure data; the actor performs the I/O. +pub(crate) struct EcdsaValidationOutcome

{ + pub(crate) ecdsa_dishonest: HashSet, + /// Failures to emit, in party iteration order. + pub(crate) failures: Vec, + pub(crate) ecdsa_passed_parties: Vec

, + pub(crate) party_addresses: HashMap, + pub(crate) party_proof_hashes: HashMap>, + pub(crate) party_public_signals: HashMap>, + pub(crate) party_proof_data: HashMap>, + /// Assembled per-party data for the consistency-check request. + pub(crate) consistency_party_data: Vec, +} + +/// Pending verification state — stored while ZK verification is in flight. +pub(crate) struct PendingVerification { + pub(crate) e3_id: E3id, + pub(crate) kind: VerificationKind, + pub(crate) ec: EventContext, + /// Parties that failed ECDSA (dishonest before ZK runs). + pub(crate) ecdsa_dishonest: HashSet, + /// Pre-dishonest parties from the dispatch (missing/incomplete proofs). + pub(crate) pre_dishonest: BTreeSet, + /// Party IDs dispatched for ZK verification (for cross-checking results). + pub(crate) dispatched_party_ids: HashSet, + /// Recovered address for each party (from ECDSA step). + pub(crate) party_addresses: HashMap, + /// Cached (proof_type, data_hash) per party — for emitting ProofVerificationPassed. + pub(crate) party_proof_hashes: HashMap>, + /// Cached (proof_type, public_signals) per party — for commitment consistency checking. + pub(crate) party_public_signals: HashMap>, + /// Parallel to `party_public_signals` — raw `proof.data` per (party, proof_type). + pub(crate) party_proof_data: HashMap>, + /// BFV preset for circuit artifact resolution. + #[allow(dead_code)] + pub(crate) params_preset: e3_fhe_params::BfvPreset, + /// Committee size for per-committee circuit artifact resolution. + #[allow(dead_code)] + pub(crate) committee_size: CiphernodesCommitteeSize, +} + +/// Pending consistency check — stored between ECDSA pass and ZK dispatch. +pub(crate) struct PendingConsistencyCheck { + pub(crate) e3_id: E3id, + pub(crate) kind: VerificationKind, + pub(crate) ec: EventContext, + /// Parties that failed ECDSA (dishonest before consistency runs). + pub(crate) ecdsa_dishonest: HashSet, + /// Pre-dishonest parties from the dispatch (missing/incomplete proofs). + pub(crate) pre_dishonest: BTreeSet, + /// Recovered address per ECDSA-passed party. + pub(crate) party_addresses: HashMap, + /// (proof_type, data_hash) per party — for ProofVerificationPassed after ZK. + pub(crate) party_proof_hashes: HashMap>, + /// (proof_type, public_signals) per party — for consistency & ZK. + pub(crate) party_public_signals: HashMap>, + /// Parallel to `party_public_signals` — raw `proof.data` per (party, proof_type). + pub(crate) party_proof_data: HashMap>, + /// Original ECDSA-passed share proofs for ZK dispatch. + pub(crate) ecdsa_passed_share_proofs: Vec, + /// Original ECDSA-passed decryption proofs for ZK dispatch. + pub(crate) ecdsa_passed_decryption_proofs: Vec, + /// BFV preset for circuit artifact resolution. + pub(crate) params_preset: e3_fhe_params::BfvPreset, + /// Committee size for per-committee circuit artifact resolution. + pub(crate) committee_size: CiphernodesCommitteeSize, +} +/// Per-party emission decision produced when tallying ZK verification results. +pub(crate) enum ZkPartyEmission { + /// Party failed ZK — attribute fault using the signed payload. + Failed { + party_id: u64, + signed: SignedProofPayload, + }, + /// Party passed ZK — emit `ProofVerificationPassed` for each cached proof. + Passed { party_id: u64 }, +} + +/// Outcome of tallying ZK verification results: the accumulated dishonest set +/// and the ordered emission decisions. +pub(crate) struct ZkTallyOutcome { + pub(crate) dishonest: BTreeSet, + pub(crate) emissions: Vec, +} + +/// Human-readable label for a verification kind (used in log lines). +pub(crate) fn label_for(kind: &VerificationKind) -> &'static str { + match kind { + VerificationKind::ShareProofs => "C2/C3", + VerificationKind::ThresholdDecryptionProofs => "C6", + VerificationKind::PkGenerationProofs => "C1", + VerificationKind::DecryptionProofs => "C4", + } +} + +/// Stateless service holding all pure share-verification business logic. +pub(crate) struct ShareVerifier; diff --git a/crates/zk-prover/src/share_verification/transitions/consistency.rs b/crates/zk-prover/src/share_verification/transitions/consistency.rs new file mode 100644 index 0000000000..49d48191cb --- /dev/null +++ b/crates/zk-prover/src/share_verification/transitions/consistency.rs @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Consistency-result filtering. + +use super::*; + +/// Filter out inconsistent parties and collect dispatched party IDs. +/// Returns `None` if all parties were filtered out (nothing to verify). +pub(crate) fn filter_consistent

( + proofs: Vec

, + inconsistent: &BTreeSet, + party_id_of: impl Fn(&P) -> u64, +) -> Option<(Vec

, HashSet)> { + let passed: Vec

= proofs + .into_iter() + .filter(|p| !inconsistent.contains(&party_id_of(p))) + .collect(); + if passed.is_empty() { + return None; + } + let ids = passed.iter().map(party_id_of).collect(); + Some((passed, ids)) +} diff --git a/crates/zk-prover/src/share_verification/transitions/ecdsa.rs b/crates/zk-prover/src/share_verification/transitions/ecdsa.rs new file mode 100644 index 0000000000..59ced9c5a6 --- /dev/null +++ b/crates/zk-prover/src/share_verification/transitions/ecdsa.rs @@ -0,0 +1,188 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Signature, signer-slot, circuit, and canonical-shape validation. + +use super::*; + +impl ShareVerifier { + /// Keccak256 over `abi_encode((proof.data, proof.public_signals))`. + pub(in crate::workflow::share_verification) fn proof_data_hash( + signed: &SignedProofPayload, + ) -> [u8; 32] { + let msg = ( + Bytes::copy_from_slice(&signed.payload.proof.data), + Bytes::copy_from_slice(&signed.payload.proof.public_signals), + ) + .abi_encode(); + keccak256(&msg).into() + } + + /// Check that a party supplied the canonical proof-type layout for this protocol phase. + /// + /// C2/C3 counts are derived from the threshold parameter preset. Variable C4b and C6 counts + /// are checked against trusted local state by their producers. This trust-boundary check + /// prevents a signed proof for another phase (or a duplicated singleton proof) from satisfying + /// the current phase merely because its self-declared [`ProofType`] maps to a valid circuit. + pub(in crate::workflow::share_verification) fn has_canonical_proof_shape( + kind: &VerificationKind, + signed_proofs: &[SignedProofPayload], + params_preset: e3_fhe_params::BfvPreset, + ) -> bool { + match kind { + VerificationKind::PkGenerationProofs => { + signed_proofs.len() == 1 + && signed_proofs[0].payload.proof_type == ProofType::C1PkGeneration + } + VerificationKind::ShareProofs => { + // Canonical order is C2a, C2b, C3a x L, C3b x L. Each C3 proof encrypts one + // modulus row of the threshold-parameter Shamir secret, even though encryption + // itself uses the paired DKG parameters. The dispatch currently carries that DKG + // preset, so recover its threshold counterpart before deriving L. + let threshold_preset = params_preset + .threshold_counterpart() + .unwrap_or(params_preset); + let num_share_rows = threshold_preset.metadata().num_moduli; + signed_proofs.len() == 2 + (2 * num_share_rows) + && signed_proofs[0].payload.proof_type == ProofType::C2aSkShareComputation + && signed_proofs[1].payload.proof_type == ProofType::C2bESmShareComputation + && signed_proofs[2..2 + num_share_rows] + .iter() + .all(|signed| signed.payload.proof_type == ProofType::C3aSkShareEncryption) + && signed_proofs[2 + num_share_rows..] + .iter() + .all(|signed| signed.payload.proof_type == ProofType::C3bESmShareEncryption) + } + VerificationKind::DecryptionProofs => { + // PartyShareDecryptionProofsToVerify has one distinguished C4a slot followed + // by one or more C4b slots. The producer checks the exact C4b count against + // `es_poly_sum`; here we bind every signed payload to its structural role because + // C4a/C4b share a CircuitName. + signed_proofs.len() >= 2 + && signed_proofs[0].payload.proof_type == ProofType::C4aSkShareDecryption + && signed_proofs[1..] + .iter() + .all(|signed| signed.payload.proof_type == ProofType::C4bESmShareDecryption) + } + VerificationKind::ThresholdDecryptionProofs => { + !signed_proofs.is_empty() + && signed_proofs.iter().all(|signed| { + signed.payload.proof_type == ProofType::C6ThresholdShareDecryption + }) + } + } + } + + /// Validate ECDSA properties for a set of signed proofs from one party: + /// 1. e3_id match + /// 2. Signature recovery (valid ECDSA) + /// 3. Recovered signer owns the canonical finalized-committee party slot + /// 4. Signer consistency (all proofs from same address) + /// 5. Circuit name matches expected ProofType circuits + pub(in crate::workflow::share_verification) fn ecdsa_validate_signed_proofs( + sender_party_id: u64, + signed_proofs: &[SignedProofPayload], + e3_id_str: &str, + label: &str, + expected_signer: Option

, + ) -> EcdsaPartyResult { + if signed_proofs.is_empty() { + info!( + "{} party {} supplied an empty signed-proof bundle", + label, sender_party_id + ); + return EcdsaPartyResult { + passed: false, + failed_payload: None, + }; + } + + let Some(expected_signer) = expected_signer else { + info!( + "{} party {} has no canonical finalized-committee slot", + label, sender_party_id + ); + return EcdsaPartyResult { + passed: false, + // The outer party id is not part of the signed payload. Its absence from the + // canonical committee is therefore a structural dispatch failure, not + // self-authenticating evidence that can safely be attributed to the signer. + failed_payload: None, + }; + }; + + let mut expected_addr: Option
= None; + + for signed in signed_proofs { + // 1. e3_id match + if signed.payload.e3_id.to_string() != e3_id_str { + info!( + "{} proof from party {} has wrong e3_id ({} vs {})", + label, sender_party_id, signed.payload.e3_id, e3_id_str + ); + return EcdsaPartyResult { + passed: false, + failed_payload: Some((signed.clone(), expected_addr)), + }; + } + + // 2. Signature recovery + match signed.recover_address() { + Ok(addr) => { + // 3. Canonical party ownership and signer consistency + if addr != expected_signer { + info!( + "{} proof signer {} does not own party {} (expected {})", + label, addr, sender_party_id, expected_signer + ); + return EcdsaPartyResult { + passed: false, + failed_payload: Some((signed.clone(), Some(addr))), + }; + } + match &expected_addr { + Some(ea) if *ea != addr => { + info!( + "{} inconsistent signer for party {}", + label, sender_party_id + ); + return EcdsaPartyResult { + passed: false, + failed_payload: Some((signed.clone(), Some(addr))), + }; + } + None => expected_addr = Some(addr), + _ => {} + } + } + Err(e) => { + info!( + "{} signature recovery failed for party {} ({:?}): {}", + label, sender_party_id, signed.payload.proof_type, e + ); + return EcdsaPartyResult { + passed: false, + failed_payload: Some((signed.clone(), expected_addr)), + }; + } + } + + // 4. Circuit name validation + let expected_circuits = signed.payload.proof_type.circuit_names(); + if !expected_circuits.contains(&signed.payload.proof.circuit) { + info!( + "{} circuit mismatch for party {}: expected {:?}, got {:?}", + label, sender_party_id, expected_circuits, signed.payload.proof.circuit + ); + return EcdsaPartyResult { + passed: false, + failed_payload: Some((signed.clone(), expected_addr)), + }; + } + } + + EcdsaPartyResult { + passed: true, + failed_payload: None, + } + } +} diff --git a/crates/zk-prover/src/share_verification/transitions/mod.rs b/crates/zk-prover/src/share_verification/transitions/mod.rs new file mode 100644 index 0000000000..4a31a88cd5 --- /dev/null +++ b/crates/zk-prover/src/share_verification/transitions/mod.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Deterministic share-verification transitions grouped by concern. + +use super::*; + +mod consistency; +mod ecdsa; +mod prepare; +mod tally; + +pub(crate) use consistency::filter_consistent; diff --git a/crates/zk-prover/src/share_verification/transitions/prepare.rs b/crates/zk-prover/src/share_verification/transitions/prepare.rs new file mode 100644 index 0000000000..698925cae4 --- /dev/null +++ b/crates/zk-prover/src/share_verification/transitions/prepare.rs @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Replay-safe batch normalization and consistency payload preparation. + +use super::*; + +impl ShareVerifier { + /// Run ECDSA validation across all parties and prepare the cached proof + /// hashes/signals/data plus the consistency-check request payload for the + /// parties that passed. Pure: no event publishing. + pub(crate) fn validate_and_prepare( + party_proofs: &[P], + e3_id_str: &str, + kind: &VerificationKind, + label: &str, + committee: Option<&[Address]>, + params_preset: e3_fhe_params::BfvPreset, + committee_size: CiphernodesCommitteeSize, + ) -> EcdsaValidationOutcome

{ + let mut ecdsa_dishonest = HashSet::new(); + let mut failures = Vec::new(); + let mut ecdsa_passed_parties = Vec::new(); + let mut party_addresses: HashMap = HashMap::new(); + + // A finalized committee is a one-to-one party-slot map. The on-chain sortition path + // guarantees unique operators, but replayed/test-produced events also cross this + // boundary. Refuse an ambiguous map rather than allowing one signer to own two slots. + let expected_committee_n = committee_size.values().n; + let committee = committee.filter(|members| { + let unique_members: HashSet

= members.iter().copied().collect(); + let is_unique = unique_members.len() == members.len(); + let has_expected_size = members.len() == expected_committee_n; + if !is_unique { + warn!( + "{} finalized committee contains duplicate addresses; rejecting proof batch", + label + ); + } + if !has_expected_size { + warn!( + "{} finalized committee has {} members, expected {}; rejecting proof batch", + label, + members.len(), + expected_committee_n + ); + } + is_unique && has_expected_size + }); + + // Collapse byte-identical replayed bundles idempotently. Two different bundles for the + // same outer party id are ambiguous and are both excluded, so a canonical party can + // contribute at most one verification result and one set of passed-proof emissions. + let mut unique_parties: Vec<&P> = Vec::new(); + let mut party_positions: HashMap = HashMap::new(); + let mut conflicting_party_ids = HashSet::new(); + for party in party_proofs { + match party_positions.get(&party.party_id()).copied() { + Some(position) if unique_parties[position] == party => { + warn!( + "{} duplicate replay for party {} ignored", + label, + party.party_id() + ); + } + Some(_) => { + warn!( + "{} conflicting proof bundles for party {}; rejecting that party", + label, + party.party_id() + ); + conflicting_party_ids.insert(party.party_id()); + } + None => { + party_positions.insert(party.party_id(), unique_parties.len()); + unique_parties.push(party); + } + } + } + + for party in unique_parties { + if conflicting_party_ids.contains(&party.party_id()) { + ecdsa_dishonest.insert(party.party_id()); + continue; + } + + let proofs = party.signed_proofs(); + if !Self::has_canonical_proof_shape(kind, &proofs, params_preset) { + info!( + "{} party {} supplied a non-canonical proof-type multiplicity/order", + label, + party.party_id() + ); + // The verification kind and outer bundle are not signed. Exclude the party, + // but do not manufacture slash evidence from an otherwise valid signed proof. + ecdsa_dishonest.insert(party.party_id()); + continue; + } + + let expected_signer = usize::try_from(party.party_id()) + .ok() + .and_then(|party_id| committee.and_then(|members| members.get(party_id))) + .copied(); + let result = Self::ecdsa_validate_signed_proofs( + party.party_id(), + &proofs, + e3_id_str, + label, + expected_signer, + ); + if result.passed { + ecdsa_passed_parties.push(party.clone()); + } else { + ecdsa_dishonest.insert(party.party_id()); + if let Some((signed, addr)) = result.failed_payload { + failures.push(EcdsaFailure { + party_id: party.party_id(), + signed, + recovered: addr, + }); + } + } + } + + // Store recovered addresses for passed parties. + for party in &ecdsa_passed_parties { + let proofs = party.signed_proofs(); + if let Some(first_signed) = proofs.first() { + if let Ok(addr) = first_signed.recover_address() { + party_addresses.insert(party.party_id(), addr); + } + } + } + + // Compute proof hashes and public signals for ECDSA-passed parties. + let mut party_proof_hashes: HashMap> = HashMap::new(); + let mut party_public_signals: HashMap> = HashMap::new(); + let mut party_raw_proof_data: HashMap> = HashMap::new(); + for party in &ecdsa_passed_parties { + let hashes: Vec<(ProofType, [u8; 32])> = party + .signed_proofs() + .iter() + .map(|signed| (signed.payload.proof_type, Self::proof_data_hash(signed))) + .collect(); + let signals: Vec<(ProofType, ArcBytes)> = party + .signed_proofs() + .iter() + .map(|signed| { + ( + signed.payload.proof_type, + signed.payload.proof.public_signals.clone(), + ) + }) + .collect(); + let datas: Vec<(ProofType, ArcBytes)> = party + .signed_proofs() + .iter() + .map(|signed| (signed.payload.proof_type, signed.payload.proof.data.clone())) + .collect(); + party_proof_hashes.insert(party.party_id(), hashes); + party_public_signals.insert(party.party_id(), signals); + party_raw_proof_data.insert(party.party_id(), datas); + } + + // Assemble consistency-check request payload. + let consistency_party_data: Vec = ecdsa_passed_parties + .iter() + .map(|party| { + let signals = party_public_signals + .get(&party.party_id()) + .cloned() + .unwrap_or_default(); + let hashes = party_proof_hashes + .get(&party.party_id()) + .cloned() + .unwrap_or_default(); + let raw_datas = party_raw_proof_data + .get(&party.party_id()) + .cloned() + .unwrap_or_default(); + let proofs = signals + .into_iter() + .zip(hashes) + .zip(raw_datas) + .map(|(((pt, ps), (_, dh)), (_, pd))| (pt, ps, dh, pd)) + .collect(); + PartyProofData { + party_id: party.party_id(), + address: party_addresses + .get(&party.party_id()) + .copied() + .unwrap_or_default(), + proofs, + } + }) + .collect(); + + EcdsaValidationOutcome { + ecdsa_dishonest, + failures, + ecdsa_passed_parties, + party_addresses, + party_proof_hashes, + party_public_signals, + party_proof_data: party_raw_proof_data, + consistency_party_data, + } + } +} diff --git a/crates/zk-prover/src/share_verification/transitions/tally.rs b/crates/zk-prover/src/share_verification/transitions/tally.rs new file mode 100644 index 0000000000..d30be6c2cd --- /dev/null +++ b/crates/zk-prover/src/share_verification/transitions/tally.rs @@ -0,0 +1,99 @@ +// SPDX-License-Identifier: LGPL-3.0-only + +//! Replay-safe ZK worker result tallying. + +use super::*; + +impl ShareVerifier { + /// Tally ZK verification results against the dispatched set: accumulate the + /// dishonest party set (including parties missing from the response) and + /// produce ordered per-party emission decisions. Pure: no event publishing. + pub(crate) fn tally_zk_results( + pre_dishonest: BTreeSet, + ecdsa_dishonest: &HashSet, + dispatched_party_ids: &HashSet, + zk_results: &[PartyVerificationResult], + ) -> ZkTallyOutcome { + let mut dishonest: BTreeSet = pre_dishonest; + dishonest.extend(ecdsa_dishonest); + + // Canonicalize the worker response before tallying. Exact duplicate responses are + // harmless replays and are collapsed; conflicting responses for one party fail closed. + // This also guarantees at most one Passed/Failed emission per canonical party. + let mut unique_results: HashMap = HashMap::new(); + let mut result_order = Vec::new(); + let mut conflicting_party_ids = HashSet::new(); + for result in zk_results { + if !dispatched_party_ids.contains(&result.sender_party_id) { + warn!( + "ZK result for party {} was not dispatched — ignoring", + result.sender_party_id + ); + continue; + } + + match unique_results.get(&result.sender_party_id).copied() { + Some(existing) if existing == result => { + warn!( + "Duplicate ZK result replay for party {} ignored", + result.sender_party_id + ); + } + Some(_) => { + warn!( + "Conflicting ZK results for party {} — treating as dishonest", + result.sender_party_id + ); + conflicting_party_ids.insert(result.sender_party_id); + } + None => { + unique_results.insert(result.sender_party_id, result); + result_order.push(result.sender_party_id); + } + } + } + + // Cross-check: every dispatched party must appear exactly once after replay collapse. + for &dispatched_pid in dispatched_party_ids { + if !unique_results.contains_key(&dispatched_pid) { + warn!( + "Party {} was dispatched for ZK verification but missing from results — treating as dishonest", + dispatched_pid + ); + dishonest.insert(dispatched_pid); + } + } + + let mut emissions = Vec::new(); + for party_id in result_order { + if conflicting_party_ids.contains(&party_id) { + dishonest.insert(party_id); + continue; + } + let Some(result) = unique_results.get(&party_id) else { + // `result_order` and `unique_results` are populated together above. Keep this + // boundary panic-free if that implementation changes later. + dishonest.insert(party_id); + continue; + }; + if !result.all_verified { + dishonest.insert(result.sender_party_id); + if let Some(ref signed) = result.failed_signed_payload { + emissions.push(ZkPartyEmission::Failed { + party_id: result.sender_party_id, + signed: signed.clone(), + }); + } + } else { + emissions.push(ZkPartyEmission::Passed { + party_id: result.sender_party_id, + }); + } + } + + ZkTallyOutcome { + dishonest, + emissions, + } + } +} diff --git a/crates/zk-prover/src/share_verification/workflow.rs b/crates/zk-prover/src/share_verification/workflow.rs new file mode 100644 index 0000000000..91be5c4001 --- /dev/null +++ b/crates/zk-prover/src/share_verification/workflow.rs @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Deterministic C1/C2/C3/C4/C6 share-proof verification workflow. +//! +//! The [`crate::actors::share_verification::ShareVerificationActor`] is a thin +//! transport shell: it owns the event bus and performs all publish/persist I/O. +//! This module owns the business logic — ECDSA validation, proof-commitment +//! hashing, consistency filtering, and ZK-result tallying — as pure functions on +//! the stateless [`ShareVerifier`] service, plus the per-E3 pending-state types. +//! It has NO actix / `BusHandle` / `Addr` concerns (tracing is allowed). + +use std::collections::{BTreeSet, HashMap, HashSet}; + +use alloy::primitives::{keccak256, Address, Bytes}; +use alloy::sol_types::SolValue; +use e3_events::{ + E3id, EventContext, PartyProofData, PartyProofsToVerify, PartyShareDecryptionProofsToVerify, + PartyVerificationResult, ProofType, Sequenced, SignedProofPayload, VerificationKind, +}; +use e3_utils::utility_types::ArcBytes; +use e3_zk_helpers::CiphernodesCommitteeSize; +use tracing::{info, warn}; + +#[path = "state.rs"] +mod state; +#[path = "transitions/mod.rs"] +mod transitions; + +pub(crate) use state::*; +pub(crate) use transitions::filter_consistent; + +#[cfg(test)] +#[path = "workflow_tests/mod.rs"] +mod tests; diff --git a/crates/zk-prover/src/share_verification/workflow_tests/ecdsa.rs b/crates/zk-prover/src/share_verification/workflow_tests/ecdsa.rs new file mode 100644 index 0000000000..4d4804d852 --- /dev/null +++ b/crates/zk-prover/src/share_verification/workflow_tests/ecdsa.rs @@ -0,0 +1,144 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[test] +fn ecdsa_passes_for_well_formed_proof() { + let s = signer(); + let e3 = e3(); + let p = signed_pk(&s, &e3, false); + let res = ShareVerifier::ecdsa_validate_signed_proofs( + 7, + &[p], + &e3.to_string(), + "C1", + Some(s.address()), + ); + assert!(res.passed); + assert!(res.failed_payload.is_none()); +} + +#[test] +fn ecdsa_fails_on_wrong_e3_id() { + let s = signer(); + let p = signed_pk(&s, &e3(), false); + let res = + ShareVerifier::ecdsa_validate_signed_proofs(7, &[p], "999/0", "C1", Some(s.address())); + assert!(!res.passed); + assert!(res.failed_payload.is_some()); +} + +#[test] +fn ecdsa_fails_on_circuit_mismatch() { + let s = signer(); + let e3 = e3(); + let p = signed_pk(&s, &e3, true); + let res = ShareVerifier::ecdsa_validate_signed_proofs( + 7, + &[p], + &e3.to_string(), + "C1", + Some(s.address()), + ); + assert!(!res.passed); +} + +#[test] +fn ecdsa_fails_on_inconsistent_signer() { + let s1 = signer(); + let s2 = signer(); + let e3 = e3(); + let p1 = signed_pk(&s1, &e3, false); + let p2 = signed_pk(&s2, &e3, false); + let res = ShareVerifier::ecdsa_validate_signed_proofs( + 7, + &[p1, p2], + &e3.to_string(), + "C1", + Some(s1.address()), + ); + assert!(!res.passed); +} + +#[test] +fn ecdsa_fails_when_signer_does_not_own_party_slot() { + let proof_signer = signer(); + let slot_owner = signer(); + let e3 = e3(); + let proof = signed_pk(&proof_signer, &e3, false); + + let result = ShareVerifier::ecdsa_validate_signed_proofs( + 1, + &[proof], + &e3.to_string(), + "C1", + Some(slot_owner.address()), + ); + + assert!(!result.passed); + let (_, recovered) = result.failed_payload.expect("attributable mismatch"); + assert_eq!(recovered, Some(proof_signer.address())); +} + +#[test] +fn ecdsa_fails_for_empty_bundle_or_missing_party_slot() { + let e3 = e3(); + let owner = signer(); + let empty = ShareVerifier::ecdsa_validate_signed_proofs( + 0, + &[], + &e3.to_string(), + "C1", + Some(owner.address()), + ); + assert!(!empty.passed); + assert!(empty.failed_payload.is_none()); + + let proof = signed_pk(&owner, &e3, false); + let missing = + ShareVerifier::ecdsa_validate_signed_proofs(3, &[proof], &e3.to_string(), "C1", None); + assert!(!missing.passed); +} + +#[test] +fn prepare_rejects_one_signer_relabelled_across_other_party_slots() { + let first = signer(); + let second = signer(); + let third = signer(); + let e3 = e3(); + let parties = vec![ + PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_pk(&first, &e3, false)], + }, + PartyProofsToVerify { + sender_party_id: 1, + // A valid proof from party 0 cannot fill party 1's slot. + signed_proofs: vec![signed_pk(&first, &e3, false)], + }, + PartyProofsToVerify { + sender_party_id: 2, + // Nor can the same signer fill any other canonical slot. + signed_proofs: vec![signed_pk(&first, &e3, false)], + }, + ]; + let committee = [first.address(), second.address(), third.address()]; + + let outcome = ShareVerifier::validate_and_prepare( + &parties, + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + + assert_eq!(outcome.ecdsa_passed_parties.len(), 1); + assert_eq!(outcome.ecdsa_passed_parties[0].sender_party_id, 0); + assert_eq!(outcome.ecdsa_dishonest, HashSet::from([1, 2])); +} diff --git a/crates/zk-prover/src/share_verification/workflow_tests/mod.rs b/crates/zk-prover/src/share_verification/workflow_tests/mod.rs new file mode 100644 index 0000000000..aa9a2f9be5 --- /dev/null +++ b/crates/zk-prover/src/share_verification/workflow_tests/mod.rs @@ -0,0 +1,95 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; +use alloy::signers::local::PrivateKeySigner; +use e3_events::{Proof, ProofPayload, ProofType}; +use e3_fhe_params::BfvPreset; + +fn signer() -> PrivateKeySigner { + PrivateKeySigner::random() +} + +fn minimum_committee(mut members: Vec
) -> Vec
{ + while members.len() < CiphernodesCommitteeSize::Minimum.values().n { + let candidate = signer().address(); + if !members.contains(&candidate) { + members.push(candidate); + } + } + members +} + +fn signed_proof( + s: &PrivateKeySigner, + e3_id: &E3id, + proof_type: ProofType, + marker: u8, +) -> SignedProofPayload { + let proof = Proof::new( + proof_type.circuit_names()[0], + ArcBytes::from_bytes(&[marker, 2, 3]), + ArcBytes::from_bytes(&[4, 5, marker]), + ); + let payload = ProofPayload { + e3_id: e3_id.clone(), + proof_type, + proof, + }; + SignedProofPayload::sign(payload, s).expect("sign") +} + +fn signed_share_bundle( + s: &PrivateKeySigner, + e3_id: &E3id, + num_share_rows: usize, +) -> Vec { + let mut proofs = vec![ + signed_proof(s, e3_id, ProofType::C2aSkShareComputation, 2), + signed_proof(s, e3_id, ProofType::C2bESmShareComputation, 3), + ]; + for _ in 0..num_share_rows { + proofs.push(signed_proof(s, e3_id, ProofType::C3aSkShareEncryption, 4)); + } + for _ in 0..num_share_rows { + proofs.push(signed_proof(s, e3_id, ProofType::C3bESmShareEncryption, 5)); + } + proofs +} + +/// Build a signed C1 (PkGeneration) proof for `party_id` under `e3_id`, +/// optionally with a deliberately wrong circuit name. +fn signed_pk(s: &PrivateKeySigner, e3_id: &E3id, wrong_circuit: bool) -> SignedProofPayload { + use e3_events::CircuitName; + let proof_type = ProofType::C1PkGeneration; + if !wrong_circuit { + return signed_proof(s, e3_id, proof_type, 1); + } + let circuit = if wrong_circuit { + CircuitName::PkBfv + } else { + proof_type.circuit_names()[0] + }; + let proof = Proof::new( + circuit, + ArcBytes::from_bytes(&[1, 2, 3]), + ArcBytes::from_bytes(&[4, 5, 6]), + ); + let payload = ProofPayload { + e3_id: e3_id.clone(), + proof_type, + proof, + }; + SignedProofPayload::sign(payload, s).expect("sign") +} + +fn e3() -> E3id { + E3id::new("1", 1) +} + +mod ecdsa; +mod shape; +mod tally; diff --git a/crates/zk-prover/src/share_verification/workflow_tests/shape.rs b/crates/zk-prover/src/share_verification/workflow_tests/shape.rs new file mode 100644 index 0000000000..8e8a768bf9 --- /dev/null +++ b/crates/zk-prover/src/share_verification/workflow_tests/shape.rs @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[test] +fn canonical_shape_rejects_cross_phase_and_singleton_multiplicity() { + let s = signer(); + let e3 = e3(); + + let c1 = signed_proof(&s, &e3, ProofType::C1PkGeneration, 1); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::PkGenerationProofs, + std::slice::from_ref(&c1), + BfvPreset::InsecureDkg512, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::PkGenerationProofs, + &[c1.clone(), c1.clone()], + BfvPreset::InsecureDkg512, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ThresholdDecryptionProofs, + std::slice::from_ref(&c1), + BfvPreset::InsecureDkg512, + )); + let c6 = signed_proof(&s, &e3, ProofType::C6ThresholdShareDecryption, 9); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ThresholdDecryptionProofs, + std::slice::from_ref(&c6), + BfvPreset::InsecureDkg512, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ThresholdDecryptionProofs, + &[], + BfvPreset::InsecureDkg512, + )); + + let share_bundle = signed_share_bundle(&s, &e3, 2); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &share_bundle, + BfvPreset::InsecureDkg512, + )); + let mut duplicate_c2a = share_bundle.clone(); + duplicate_c2a.insert(1, duplicate_c2a[0].clone()); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &duplicate_c2a, + BfvPreset::InsecureDkg512, + )); + + let secure_share_bundle = signed_share_bundle(&s, &e3, 3); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &secure_share_bundle, + BfvPreset::SecureThreshold8192, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &share_bundle, + BfvPreset::SecureThreshold8192, + )); + + let c4_bundle = vec![ + signed_proof(&s, &e3, ProofType::C4aSkShareDecryption, 6), + signed_proof(&s, &e3, ProofType::C4bESmShareDecryption, 7), + ]; + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::DecryptionProofs, + &c4_bundle, + BfvPreset::InsecureDkg512, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::DecryptionProofs, + &c4_bundle[1..], + BfvPreset::InsecureDkg512, + )); + let mut extra_c4b = c4_bundle.clone(); + extra_c4b.push(c4_bundle[1].clone()); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::DecryptionProofs, + &extra_c4b, + BfvPreset::InsecureDkg512, + )); + let mut wrong_c4_tail = c4_bundle.clone(); + wrong_c4_tail.push(c6); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::DecryptionProofs, + &wrong_c4_tail, + BfvPreset::InsecureDkg512, + )); +} + +#[test] +fn share_shape_uses_threshold_secret_rows_when_dispatch_carries_dkg_preset() { + let s = signer(); + let e3 = e3(); + + // Production dispatch carries the share-encryption (DKG) preset, but C3 requests are + // generated from rows of the paired threshold-parameter Shamir secret. + let insecure_production_bundle = signed_share_bundle(&s, &e3, 2); + assert_eq!(BfvPreset::InsecureDkg512.metadata().num_moduli, 1); + assert_eq!(BfvPreset::InsecureThreshold512.metadata().num_moduli, 2); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &insecure_production_bundle, + BfvPreset::InsecureDkg512, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &signed_share_bundle(&s, &e3, 1), + BfvPreset::InsecureDkg512, + )); + + let secure_production_bundle = signed_share_bundle(&s, &e3, 3); + assert_eq!(BfvPreset::SecureDkg8192.metadata().num_moduli, 2); + assert_eq!(BfvPreset::SecureThreshold8192.metadata().num_moduli, 3); + assert!(ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &secure_production_bundle, + BfvPreset::SecureDkg8192, + )); + assert!(!ShareVerifier::has_canonical_proof_shape( + &VerificationKind::ShareProofs, + &signed_share_bundle(&s, &e3, 2), + BfvPreset::SecureDkg8192, + )); +} + +#[test] +fn prepare_excludes_wrong_phase_without_creating_slash_evidence() { + let s = signer(); + let e3 = e3(); + let parties = [PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_proof( + &s, + &e3, + ProofType::C6ThresholdShareDecryption, + 9, + )], + }]; + let committee = minimum_committee(vec![s.address()]); + + let outcome = ShareVerifier::validate_and_prepare( + &parties, + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + + assert!(outcome.ecdsa_passed_parties.is_empty()); + assert_eq!(outcome.ecdsa_dishonest, HashSet::from([0])); + assert!(outcome.failures.is_empty()); +} + +#[test] +fn prepare_collapses_identical_party_replay_and_rejects_conflict() { + let s = signer(); + let e3 = e3(); + let committee = minimum_committee(vec![s.address()]); + let party = PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 1)], + }; + + let replayed = ShareVerifier::validate_and_prepare( + &[party.clone(), party.clone()], + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + assert_eq!(replayed.ecdsa_passed_parties, vec![party.clone()]); + assert!(replayed.ecdsa_dishonest.is_empty()); + assert_eq!(replayed.consistency_party_data.len(), 1); + + let conflicting = PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 2)], + }; + let conflict = ShareVerifier::validate_and_prepare( + &[party, conflicting], + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + assert!(conflict.ecdsa_passed_parties.is_empty()); + assert_eq!(conflict.ecdsa_dishonest, HashSet::from([0])); + assert!(conflict.failures.is_empty()); +} diff --git a/crates/zk-prover/src/share_verification/workflow_tests/tally.rs b/crates/zk-prover/src/share_verification/workflow_tests/tally.rs new file mode 100644 index 0000000000..7994abfdd8 --- /dev/null +++ b/crates/zk-prover/src/share_verification/workflow_tests/tally.rs @@ -0,0 +1,140 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +use super::*; + +#[test] +fn prepare_rejects_ambiguous_committee_where_one_signer_owns_multiple_slots() { + let s = signer(); + let e3 = e3(); + let parties = vec![ + PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 1)], + }, + PartyProofsToVerify { + sender_party_id: 1, + signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 2)], + }, + ]; + let ambiguous_committee = [s.address(), s.address(), signer().address()]; + + let outcome = ShareVerifier::validate_and_prepare( + &parties, + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&ambiguous_committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + + assert!(outcome.ecdsa_passed_parties.is_empty()); + assert_eq!(outcome.ecdsa_dishonest, HashSet::from([0, 1])); + assert!(outcome.failures.is_empty()); +} + +#[test] +fn prepare_rejects_committee_with_wrong_circuit_dimension() { + let s = signer(); + let e3 = e3(); + let parties = [PartyProofsToVerify { + sender_party_id: 0, + signed_proofs: vec![signed_proof(&s, &e3, ProofType::C1PkGeneration, 1)], + }]; + let undersized_committee = [s.address()]; + + let outcome = ShareVerifier::validate_and_prepare( + &parties, + &e3.to_string(), + &VerificationKind::PkGenerationProofs, + "C1", + Some(&undersized_committee), + BfvPreset::InsecureDkg512, + CiphernodesCommitteeSize::Minimum, + ); + + assert!(outcome.ecdsa_passed_parties.is_empty()); + assert_eq!(outcome.ecdsa_dishonest, HashSet::from([0])); + assert!(outcome.failures.is_empty()); +} + +#[test] +fn filter_consistent_drops_inconsistent_and_returns_ids() { + let proofs = vec![1u64, 2, 3]; + let inconsistent: BTreeSet = [2].into_iter().collect(); + let (passed, ids) = filter_consistent(proofs, &inconsistent, |p| *p).expect("some"); + assert_eq!(passed, vec![1, 3]); + assert!(ids.contains(&1) && ids.contains(&3) && !ids.contains(&2)); +} + +#[test] +fn filter_consistent_returns_none_when_all_filtered() { + let proofs = vec![1u64, 2]; + let inconsistent: BTreeSet = [1, 2].into_iter().collect(); + assert!(filter_consistent(proofs, &inconsistent, |p| *p).is_none()); +} + +#[test] +fn tally_marks_missing_dispatched_party_dishonest() { + let dispatched: HashSet = [1, 2].into_iter().collect(); + let ecdsa: HashSet = HashSet::new(); + // No ZK results at all → both dispatched parties are missing → dishonest. + let out = ShareVerifier::tally_zk_results(BTreeSet::new(), &ecdsa, &dispatched, &[]); + assert!(out.dishonest.contains(&1)); + assert!(out.dishonest.contains(&2)); + assert!(out.emissions.is_empty()); +} + +#[test] +fn tally_collapses_identical_result_replay() { + let dispatched = HashSet::from([1]); + let result = PartyVerificationResult { + sender_party_id: 1, + all_verified: true, + failed_signed_payload: None, + recovered_address: None, + }; + + let out = ShareVerifier::tally_zk_results( + BTreeSet::new(), + &HashSet::new(), + &dispatched, + &[result.clone(), result], + ); + + assert!(out.dishonest.is_empty()); + assert_eq!(out.emissions.len(), 1); + assert!(matches!( + out.emissions[0], + ZkPartyEmission::Passed { party_id: 1 } + )); +} + +#[test] +fn tally_rejects_conflicting_results_for_one_party() { + let dispatched = HashSet::from([1]); + let passed = PartyVerificationResult { + sender_party_id: 1, + all_verified: true, + failed_signed_payload: None, + recovered_address: None, + }; + let failed = PartyVerificationResult { + all_verified: false, + ..passed.clone() + }; + + let out = ShareVerifier::tally_zk_results( + BTreeSet::new(), + &HashSet::new(), + &dispatched, + &[passed, failed], + ); + + assert_eq!(out.dishonest, BTreeSet::from([1])); + assert!(out.emissions.is_empty()); +} diff --git a/crates/zk-prover/src/workflow.rs b/crates/zk-prover/src/workflow.rs new file mode 100644 index 0000000000..dc3209118e --- /dev/null +++ b/crates/zk-prover/src/workflow.rs @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: LGPL-3.0-only +// +// This file is provided WITHOUT ANY WARRANTY; +// without even the implied warranty of MERCHANTABILITY +// or FITNESS FOR A PARTICULAR PURPOSE. + +//! Compatibility view of workflows stored with their proof capabilities. + +#[path = "proof_request/workflow.rs"] +pub(crate) mod proof_request; +#[path = "share_verification/workflow.rs"] +pub(crate) mod share_verification; diff --git a/crates/zk-prover/src/actors/zk_actor.rs b/crates/zk-prover/src/zk_actor.rs similarity index 100% rename from crates/zk-prover/src/actors/zk_actor.rs rename to crates/zk-prover/src/zk_actor.rs From 7e1ad353e1a5b6a873de9f7f6eac635c147d44c5 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 14 Jul 2026 22:26:23 +0500 Subject: [PATCH 56/62] fix(aggregator): normalize expelled committee addresses --- agent/flow-trace/00_INDEX.md | 2 +- agent/flow-trace/04_DKG_AND_COMPUTATION.md | 2 + .../flow-trace/05_FAILURE_REFUND_SLASHING.md | 3 +- .../src/public_key_aggregation/effects/mod.rs | 3 +- .../src/public_key_aggregation/handlers.rs | 4 +- .../public_key_aggregation/keyshare_buffer.rs | 43 ++++++++++++++---- .../src/public_key_aggregation/transitions.rs | 22 ++++++--- .../public_key_aggregation/workflow_tests.rs | 45 +++++++++++++++---- 8 files changed, 97 insertions(+), 27 deletions(-) diff --git a/agent/flow-trace/00_INDEX.md b/agent/flow-trace/00_INDEX.md index 3607eb30d0..6b85381ab2 100644 --- a/agent/flow-trace/00_INDEX.md +++ b/agent/flow-trace/00_INDEX.md @@ -191,7 +191,7 @@ _Found during source-code cross-referencing of these trace documents._ | 3 | **`gracePeriod` is dead code** | Medium | `gracePeriod` is stored and validated during config updates but never actually used in any timeout check. Either the deadlines already bake in sufficient buffer, or this is a missing feature. | | 4 | **`activate` CLI command is misleading** | Low | Named "activate" but actually calls "register" — will fail for already-registered operators. There's no standalone way to trigger re-evaluation of active status; instead, `_updateOperatorStatus()` runs automatically inside `addTicketBalance()`, `bondLicense()`, etc. | | 5 | **Active-job load balancing bug fixed** | Info | The Rust `NodeStateStore.available_tickets()` subtracts `active_jobs` from total tickets, reducing the chance of busy nodes being selected for new E3s. Previously, the `Sortition` actor's `Handler` was missing match arms for `E3Failed` and `E3StageChanged`, causing these events to fall to the default `_ => ()` — the typed handlers for decrementing jobs were dead code. This has been fixed: E3Failed and E3StageChanged are now routed to their handlers, and `finalized_committees` is cleaned up in `decrement_jobs_for_e3` to prevent unbounded memory growth. | -| 6 | **Committee member expulsion** | Info | `SlashingManager` can call `expelCommitteeMember()` mid-DKG. The `Sortition` actor enriches the raw `CommitteeMemberExpelled` event with the expelled member's `party_id` (resolved from its stored `Committee` list) and re-publishes it. `ThresholdKeyshare` then uses the enriched `party_id` to update its collectors, potentially completing DKG with fewer parties. `ThresholdKeyshare` itself does not hold committee state. | +| 6 | **Committee member expulsion** | Resolved | `SlashingManager` can call `expelCommitteeMember()` mid-DKG. The `Sortition` actor enriches the raw `CommitteeMemberExpelled` event with the expelled member's `party_id` (resolved from its stored `Committee` list) and re-publishes it. `ThresholdKeyshare` uses the enriched `party_id` to update its collectors. The public-key keyshare gate and aggregation reducer normalize both expulsion records and self-reported keyshare nodes to `alloy::Address`, so an expelled member cannot re-enter by changing address casing. `ThresholdKeyshare` itself does not hold committee state. | | 7 | **ProofRequestActor failure bridge fixed** | Info | `ProofRequestActor` no longer leaves proof publication suppressed under log-only "will not be published" exits. `ComputeRequestError` and local proof-signing failures for DKG-path proofs (`C0` through `C5`) now emit `E3Failed { failed_at_stage: CommitteeFinalized, reason: DKGInvalidShares }`, while decryption-path proofs (`C6` and `C7`) emit `E3Failed { failed_at_stage: CiphertextReady, reason: DecryptionInvalidShares }`. | | 8 | **Settlement receipts isolated** | Resolved | Durable EVM receipts such as `RewardCredited` and `RewardClaimed` are global audit/projection facts. The E3 router no longer sends them into a completed per-E3 context, so reward fan-out cannot reopen a finished E3 or produce false `AlreadyCompleted` failures. | | 9 | **Replay-safe compute effects** | Resolved | `ComputeEffectGate` subscribes before EventStore replay, buffers `ComputeRequest`s while effects are disabled, deduplicates equivalent requests, prefers the newest hydrated retry, cancels terminal E3 work, and releases pending effects only after `EffectsEnabled`. This closes the mid-E3 compute-loss window without changing durable event order. | diff --git a/agent/flow-trace/04_DKG_AND_COMPUTATION.md b/agent/flow-trace/04_DKG_AND_COMPUTATION.md index bbac1c4da4..e2f9d32eaf 100644 --- a/agent/flow-trace/04_DKG_AND_COMPUTATION.md +++ b/agent/flow-trace/04_DKG_AND_COMPUTATION.md @@ -561,6 +561,8 @@ ThresholdKeyshare receives AllThresholdSharesCollected │ ├─ KeyshareCreatedFilterBuffer gates events: │ └─ Only accepts KeyshareCreated from verified committee members + │ └─ Compares committee, keyshare, and expulsion identities as parsed EVM addresses; + │ EIP-55 casing differences cannot bypass an expulsion or its buffered-share purge │ └─ Buffers until BOTH CommitteeFinalized and AggregatorChanged(is_aggregator=true) │ └─ On expulsion-driven handoff, the next active aggregator flushes its existing buffer │ diff --git a/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md b/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md index cc1e0a6056..9c482ced07 100644 --- a/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md +++ b/agent/flow-trace/05_FAILURE_REFUND_SLASHING.md @@ -1076,7 +1076,8 @@ When CommitteeMemberExpelled event arrives from EVM: │ ├─ KeyshareCreatedFilterBuffer (aggregator): │ ├─ Only processes raw events (party_id: None) -│ └─ Removes expelled node from committee filter set +│ └─ Stores the expelled node as `alloy::Address` and removes/blocks keyshares by parsed +│ address, so differently cased self-reported node strings cannot bypass expulsion │ └─ When E3Failed(timeout) / E3StageChanged(Complete) arrives: │ diff --git a/crates/aggregator/src/public_key_aggregation/effects/mod.rs b/crates/aggregator/src/public_key_aggregation/effects/mod.rs index 23b01b30d7..41abc3cca5 100644 --- a/crates/aggregator/src/public_key_aggregation/effects/mod.rs +++ b/crates/aggregator/src/public_key_aggregation/effects/mod.rs @@ -7,6 +7,7 @@ //! Effect execution for public-key aggregation. use super::*; +use alloy::primitives::Address; mod aggregate_dkg_proofs; mod aggregate_public_key; @@ -18,7 +19,7 @@ mod verify_key_proofs; impl PublicKeyAggregator { pub fn handle_member_expelled( &mut self, - node: &str, + node: Address, ec: &EventContext, ) -> Result<()> { self.state.try_mutate(ec, |state| { diff --git a/crates/aggregator/src/public_key_aggregation/handlers.rs b/crates/aggregator/src/public_key_aggregation/handlers.rs index d59c2db9d7..9dffda733d 100644 --- a/crates/aggregator/src/public_key_aggregation/handlers.rs +++ b/crates/aggregator/src/public_key_aggregation/handlers.rs @@ -41,7 +41,7 @@ impl Handler for PublicKeyAggregator { return; } - let node_addr = data.node.to_string(); + let node_addr = data.node; if data.e3_id != self.e3_id { error!("Wrong e3_id sent to PublicKeyAggregator for expulsion. This should not happen."); @@ -58,7 +58,7 @@ impl Handler for PublicKeyAggregator { Some(PublicKeyAggregatorState::Collecting { .. }) ); - self.handle_member_expelled(&node_addr, &ec)?; + self.handle_member_expelled(node_addr, &ec)?; // If we just transitioned to VerifyingC1, dispatch C1 verification // using the c1_proofs now stored in the VerifyingC1 state (already diff --git a/crates/aggregator/src/public_key_aggregation/keyshare_buffer.rs b/crates/aggregator/src/public_key_aggregation/keyshare_buffer.rs index 8a608eda50..8af5b9bab9 100644 --- a/crates/aggregator/src/public_key_aggregation/keyshare_buffer.rs +++ b/crates/aggregator/src/public_key_aggregation/keyshare_buffer.rs @@ -21,7 +21,7 @@ pub struct KeyshareCreatedFilterBuffer { /// accepting a real member under another member's `party_id` poisons the DKG roster. committee: Option>, buffer: Vec, - expelled_nodes: HashSet, + expelled_nodes: HashSet
, is_aggregator: bool, } @@ -37,6 +37,11 @@ fn committee_member_matches(committee: &[String], party_id: u64, node: &str) -> .is_ok_and(|address| address == expected) } +fn address_matches(node: &str, expected: Address) -> bool { + node.parse::
() + .is_ok_and(|address| address == expected) +} + impl KeyshareCreatedFilterBuffer { pub fn new(dest: Addr) -> Self { Self { @@ -58,7 +63,10 @@ impl KeyshareCreatedFilterBuffer { match event.get_data() { InterfoldEventData::KeyshareCreated(data) if committee_member_matches(committee, data.party_id, &data.node) - && !self.expelled_nodes.contains(&data.node) => + && !data + .node + .parse::
() + .is_ok_and(|node| self.expelled_nodes.contains(&node)) => { self.dest.do_send(event); } @@ -93,7 +101,10 @@ impl Handler for KeyshareCreatedFilterBuffer { Some(committee) if self.is_aggregator && committee_member_matches(committee, data.party_id, &data.node) - && !self.expelled_nodes.contains(&data.node) => + && !data + .node + .parse::
() + .is_ok_and(|node| self.expelled_nodes.contains(&node)) => { self.dest.do_send(msg); } @@ -102,7 +113,10 @@ impl Handler for KeyshareCreatedFilterBuffer { } Some(committee) if committee_member_matches(committee, data.party_id, &data.node) - && !self.expelled_nodes.contains(&data.node) => + && !data + .node + .parse::
() + .is_ok_and(|node| self.expelled_nodes.contains(&node)) => { self.buffer.push(msg); } @@ -119,12 +133,13 @@ impl Handler for KeyshareCreatedFilterBuffer { return; } - let node_addr = data.node.to_string(); - self.expelled_nodes.insert(node_addr.clone()); + let node_addr = data.node; + self.expelled_nodes.insert(node_addr); self.buffer.retain(|event| { !matches!( event.get_data(), - InterfoldEventData::KeyshareCreated(share) if share.node == node_addr + InterfoldEventData::KeyshareCreated(share) + if address_matches(&share.node, node_addr) ) }); @@ -166,7 +181,8 @@ impl Handler for KeyshareCreatedFilterBuffer { #[cfg(test)] mod tests { - use super::committee_member_matches; + use super::{address_matches, committee_member_matches}; + use alloy::primitives::Address; const A: &str = "0x1111111111111111111111111111111111111111"; const B: &str = "0x2222222222222222222222222222222222222222"; @@ -191,4 +207,15 @@ mod tests { "0xabcd000000000000000000000000000000000000" )); } + + #[test] + fn expelled_address_comparison_is_case_insensitive() { + let expelled = "0xAbCd000000000000000000000000000000000000" + .parse::
() + .unwrap(); + assert!(address_matches( + "0xabcd000000000000000000000000000000000000", + expelled + )); + } } diff --git a/crates/aggregator/src/public_key_aggregation/transitions.rs b/crates/aggregator/src/public_key_aggregation/transitions.rs index c73d8691f4..8d06aff00e 100644 --- a/crates/aggregator/src/public_key_aggregation/transitions.rs +++ b/crates/aggregator/src/public_key_aggregation/transitions.rs @@ -191,7 +191,7 @@ impl PublicKeyAggregation { /// collections aligned and transitioning to `VerifyingC1` when enough keyshares remain. pub(crate) fn handle_member_expelled( mut state: PublicKeyAggregatorState, - node: &str, + node: Address, ) -> Result { let PublicKeyAggregatorState::Collecting { threshold_n, @@ -208,17 +208,29 @@ impl PublicKeyAggregation { return Ok(state); }; - let node_str = node.to_string(); - // Find the expelled node's index in submission_order and remove from // all parallel collections so they stay aligned. - if let Some(idx) = submission_order.iter().position(|(_, n, _)| n == &node_str) { + if let Some(idx) = submission_order.iter().position(|(_, candidate, _)| { + candidate + .parse::
() + .is_ok_and(|candidate| candidate == node) + }) { let (_, _, expelled_keyshare) = submission_order.remove(idx); keyshares.remove(&expelled_keyshare); c1_proofs.remove(idx); } - nodes.remove(&node_str); + let expelled_node = nodes + .iter() + .find(|candidate| { + candidate + .parse::
() + .is_ok_and(|candidate| candidate == node) + }) + .cloned(); + if let Some(expelled_node) = expelled_node { + nodes.remove(&expelled_node); + } if *threshold_n > 0 { *threshold_n -= 1; diff --git a/crates/aggregator/src/public_key_aggregation/workflow_tests.rs b/crates/aggregator/src/public_key_aggregation/workflow_tests.rs index 5789834596..961c4b119e 100644 --- a/crates/aggregator/src/public_key_aggregation/workflow_tests.rs +++ b/crates/aggregator/src/public_key_aggregation/workflow_tests.rs @@ -152,18 +152,28 @@ fn select_honest_set_fails_when_at_or_below_threshold_m() { #[test] fn handle_member_expelled_removes_and_reduces_threshold() { let mut state = collecting(3, 1); - for pid in 0..2u64 { + let nodes = [ + "0xabcdef0000000000000000000000000000000000", + "0x2222222222222222222222222222222222222222", + ]; + for (pid, node) in nodes.into_iter().enumerate() { state = PublicKeyAggregation::add_keyshare( state, ks(pid as u8), - format!("node-{pid}"), - pid, + node.to_owned(), + pid as u64, None, ) .unwrap(); } // expel node-0; threshold_n 3 -> 2, keyshares now 1 (< 2) -> stays Collecting - let state = PublicKeyAggregation::handle_member_expelled(state, "node-0").unwrap(); + let state = PublicKeyAggregation::handle_member_expelled( + state, + "0xAbCdEf0000000000000000000000000000000000" + .parse() + .unwrap(), + ) + .unwrap(); match state { PublicKeyAggregatorState::Collecting { threshold_n, @@ -173,7 +183,7 @@ fn handle_member_expelled_removes_and_reduces_threshold() { } => { assert_eq!(threshold_n, 2); assert_eq!(submission_order.len(), 1); - assert!(!nodes.contains(&"node-0".to_string())); + assert!(!nodes.contains(&"0xabcdef0000000000000000000000000000000000".to_string())); } _ => panic!("expected Collecting"), } @@ -191,13 +201,30 @@ fn handle_member_expelled_transitions_when_enough_remain() { keyshares: OrderedSet::from(vec![ks(10), ks(11)]), c1_proofs: vec![None, None], seed: Seed([0u8; 32]), - nodes: OrderedSet::from(vec!["node-0".to_string(), "node-1".to_string()]), + nodes: OrderedSet::from(vec![ + "0xabcdef0000000000000000000000000000000000".to_string(), + "0x2222222222222222222222222222222222222222".to_string(), + ]), submission_order: vec![ - (0, "node-0".to_string(), ks(10)), - (1, "node-1".to_string(), ks(11)), + ( + 0, + "0xabcdef0000000000000000000000000000000000".to_string(), + ks(10), + ), + ( + 1, + "0x2222222222222222222222222222222222222222".to_string(), + ks(11), + ), ], }; - let next = PublicKeyAggregation::handle_member_expelled(state, "node-0").unwrap(); + let next = PublicKeyAggregation::handle_member_expelled( + state, + "0xAbCdEf0000000000000000000000000000000000" + .parse() + .unwrap(), + ) + .unwrap(); match next { PublicKeyAggregatorState::VerifyingC1 { circuit_committee_n, From 4de3ec189ef596bbbb805520377bb4ed2a5aff09 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 14 Jul 2026 22:30:26 +0500 Subject: [PATCH 57/62] fix(evm): skip unsupported E3 versions safely --- agent/CRATES_ARCHITECTURE.md | 2 +- agent/flow-trace/00_INDEX.md | 1 + .../flow-trace/03_E3_REQUEST_AND_COMMITTEE.md | 4 + crates/evm/src/event_decoding/actor.rs | 91 ++++++++-- crates/evm/src/interfold/events.rs | 164 ++++++++++-------- crates/evm/src/interfold/reader.rs | 2 +- 6 files changed, 177 insertions(+), 87 deletions(-) diff --git a/agent/CRATES_ARCHITECTURE.md b/agent/CRATES_ARCHITECTURE.md index d621151213..4382c83251 100644 --- a/agent/CRATES_ARCHITECTURE.md +++ b/agent/CRATES_ARCHITECTURE.md @@ -837,7 +837,7 @@ boundary, so they cannot all prove completion or persist recovery intent. | `e3-data` | Serve typed repository reads/writes and append-only event-log/index records. | Sled/in-memory stores, log handles, batch writes, and flush failure state. | Acknowledged sync/batch writes flush before success; decode corruption and recorded write failures fail closed. | Repository/store factories; must not decide committees, proofs, or lifecycle transitions. | | `e3-sync` | Reconstruct actor state and reconcile EVM/network history before live mode. | Startup plan, disk-backed local replay runs, and bounded reconciled-history vectors; depends on repositories, EventBus, EVM, and net adapters. | Schema is checked before state-writing actors; HLC includes post-snapshot history; replay is global-HLC ordered with acknowledged subscriber acceptance; Effects/history/SyncEnded phases are downstream-acknowledged; history gaps or bounded-net-sync failure abort startup. | Historical collectors/planners; must not submit live transactions. | | `e3-net` | Translate bounded libp2p traffic and serve gossip, DHT, and historical sync. | Swarm, Kademlia records, peer/transport status, channels, startup buffer, and document interests. | Signed gossip is type-allowlisted; decodes, startup backlog, and sync fetches are bounded; metadata must match DHT payload. Errors fail readiness or stop the affected ingress loop. | `NetInterface` and pure translation services; must not own E3 transitions or infer committee authority from PeerId alone. | -| `e3-evm` | Read chain history under the configured confirmation policy and submit typed contract transactions. | Per-chain gateways, provider handles, chain buffers, nonce mutexes, and slash replay gate. | Malformed logs and reverted receipts fail; nonce allocation is serialized in-process; no durable transaction outbox or full reorg rollback exists. | Provider/contract helpers; must not own off-chain proof policy. | +| `e3-evm` | Read chain history under the configured confirmation policy and submit typed contract transactions. | Per-chain gateways, provider handles, chain buffers, nonce mutexes, and slash replay gate. | Malformed logs and reverted receipts fail. A well-formed `E3Requested` with an unsupported committee/preset enum is marked processed and skipped so an older node stays live without participating. Nonce allocation is serialized in-process; no durable transaction outbox or full reorg rollback exists. | Provider/contract helpers; must not own off-chain proof policy. | | `e3-request` | Route E3-scoped events and enforce lifecycle progress. | `E3Router`, lifecycle state, typed `(E3, recipient)` buffers, and request actor contexts; depends on event and protocol actor APIs. | Legal progress is monotonic; buffered history precedes the recipient-creating event; terminal teardown purges absent-role buffers. Active buffer size and child `do_send` remain residual risks. | Domain lifecycle/routing functions; must not implement storage, network framing, or contract decoding. | | `e3-sortition` | Track registry/tickets and derive canonical selection/committee observations. | Node registry, ticket state, selector backend, and chain-derived committee state. | On-chain ordering is authoritative; terminal cleanup releases local participation state and removes durable finalized-committee and pending-expulsion records. | Sortition backend; must not construct cryptographic proofs. | | `e3-keyshare` | Coordinate request-local DKG, shares, and decryption work. | Threshold keyshare actor state and repositories; depends on FHE/ZK services and the event bus. | Party IDs index the canonical committee; each recipient gets C2a/C2b singletons and C3a/C3b per threshold Shamir row; resumable determined outputs redrive only after `EffectsEnabled`. | Cryptographic backend/task pool; must not own transport frames or ABI decoding. | diff --git a/agent/flow-trace/00_INDEX.md b/agent/flow-trace/00_INDEX.md index 6b85381ab2..335bc90352 100644 --- a/agent/flow-trace/00_INDEX.md +++ b/agent/flow-trace/00_INDEX.md @@ -198,3 +198,4 @@ _Found during source-code cross-referencing of these trace documents._ | 10 | **Restarted active aggregator decryption recovery** | Resolved | A node killed after key publication but before ciphertext publication could miss the one-shot `AggregatorChanged` role event before its plaintext buffer existed, lack recovered committee dependencies when ciphertext arrived, or rehydrate without the per-E3 `CommitmentConsistencyChecker` needed to answer C6 verification gates. `ThresholdPlaintextAggregatorExtension` records aggregator role, remembers pending ciphertext until committee facts arrive, recovers full committee data from public-key state/replayed public events, recovers the honest subset from public-key or threshold-keyshare state, `CommitmentConsistencyCheckerExtension` recreates its per-E3 checker from recovered meta, and restart recovery never blocks the router on synchronous store reads. | | 11 | **Large EventStore replay startup failure** | Resolved | Restart replay now awaits each EventBus handler result before submitting the next persisted event. A backlog larger than the 2,560-entry EventBus mailbox no longer fails startup with `SendError::Full`; progress is logged every 10,000 EventBus-handled events. The EventBus still fans out with unacknowledged `do_send`, and replay still materializes the full range in memory, so end-to-end queue and memory bounds remain open. | | 12 | **Canonical DKG H/N aggregation shape** | Resolved | The Rust DKG aggregation path no longer asserts that honest proof count `H` equals full committee size `N`. Every supported preset has `H < N`; validation now requires exactly `H` NodeFold proofs with unique in-range party IDs and exactly `N` ordered committee addresses, matching the producer and compiled Noir witness. The removed assertion affected debug/test binaries; circuit semantics and public inputs are unchanged. | +| 13 | **Forward-compatible E3 version skew** | Resolved | A well-formed `E3Requested` carrying an unsupported committee-size or BFV-preset enum is now classified as a benign skip for that older binary. The parser emits an internal `Processed` marker so historical ordering and startup can advance without dispatching the E3. ABI decode failures and missing indexed topics remain rejected and fail the chain pipeline closed. | diff --git a/agent/flow-trace/03_E3_REQUEST_AND_COMMITTEE.md b/agent/flow-trace/03_E3_REQUEST_AND_COMMITTEE.md index 0d99dcd705..b30c5ea659 100644 --- a/agent/flow-trace/03_E3_REQUEST_AND_COMMITTEE.md +++ b/agent/flow-trace/03_E3_REQUEST_AND_COMMITTEE.md @@ -122,6 +122,10 @@ When the running ciphernodes detect `E3Requested` and `CommitteeRequested` event ``` InterfoldSolReader decodes IInterfold::E3Requested log │ +├─ If the ABI log is well-formed but its committee-size or BFV-preset enum is newer than this +│ binary supports, records the provider log as internally processed and skips participation; +│ historical ordering advances, while malformed ABI data still fails chain ingestion closed +│ ├─ Publishes InterfoldEvent::E3Requested { │ e3_id, threshold_m, threshold_n, │ seed, params, error_size, esi_per_ct diff --git a/crates/evm/src/event_decoding/actor.rs b/crates/evm/src/event_decoding/actor.rs index c8f0f43cb2..fa1b705361 100644 --- a/crates/evm/src/event_decoding/actor.rs +++ b/crates/evm/src/event_decoding/actor.rs @@ -15,10 +15,16 @@ use crate::domain::log_timestamp::from_log_chain_id_to_ts; use crate::messages::{EvmEvent, EvmEventProcessor, EvmLog, EvmLogRejected, InterfoldEvmEvent}; pub type ExtractorFn = fn(&LogData, &[B256], u64) -> Option; +pub type VersionAwareExtractorFn = fn(&LogData, &[B256], u64) -> Result>; + +enum Extractor { + Strict(ExtractorFn), + VersionAware(VersionAwareExtractorFn), +} pub struct EvmParser { next: EvmEventProcessor, - extractor: ExtractorFn, + extractor: Extractor, } impl Actor for EvmParser { @@ -32,23 +38,49 @@ impl EvmParser { pub fn new(next: &EvmEventProcessor, extractor: ExtractorFn) -> Self { Self { next: next.clone(), - extractor, + extractor: Extractor::Strict(extractor), + } + } + + pub fn new_version_aware( + next: &EvmEventProcessor, + extractor: VersionAwareExtractorFn, + ) -> Self { + Self { + next: next.clone(), + extractor: Extractor::VersionAware(extractor), } } } -fn parse_log(log: EvmLog, extractor: ExtractorFn) -> Result { +fn parse_log(log: EvmLog, extractor: &Extractor) -> Result> { let block = log.log.block_number.context( "provider log is missing its block number; pending or malformed logs cannot be ordered", )?; let log_index = log.log.log_index.context( "provider log is missing its log index; malformed logs cannot be ordered deterministically", )?; - let event = extractor(log.log.data(), log.log.topics(), log.chain_id).context( - "contract log matched a configured address but could not be decoded; refusing to advance", - )?; + let event = match extractor { + Extractor::Strict(extractor) => Some( + extractor(log.log.data(), log.log.topics(), log.chain_id).context( + "contract log matched a configured address but could not be decoded; refusing to advance", + )?, + ), + Extractor::VersionAware(extractor) => { + extractor(log.log.data(), log.log.topics(), log.chain_id)? + } + }; + let Some(event) = event else { + return Ok(None); + }; let timestamp = from_log_chain_id_to_ts(log.timestamp, log_index, log.chain_id); - Ok(EvmEvent::new(log.id, event, block, timestamp, log.chain_id)) + Ok(Some(EvmEvent::new( + log.id, + event, + block, + timestamp, + log.chain_id, + ))) } impl Handler for EvmParser { @@ -59,8 +91,12 @@ impl Handler for EvmParser { debug!("processing event({})", msg.get_id()); let id = log.id; let chain_id = log.chain_id; - match parse_log(log, self.extractor) { - Ok(event) => self.next.do_send(InterfoldEvmEvent::Event(event)), + match parse_log(log, &self.extractor) { + Ok(Some(event)) => self.next.do_send(InterfoldEvmEvent::Event(event)), + Ok(None) => { + debug!(%id, chain_id, "Skipping unsupported EVM event and advancing historical ordering"); + self.next.do_send(InterfoldEvmEvent::Processed(id)); + } Err(parse_error) => { error!( %id, @@ -99,6 +135,10 @@ mod tests { None } + fn skipped_extractor(_: &LogData, _: &[B256], _: u64) -> Result> { + Ok(None) + } + fn log(block_number: Option, log_index: Option) -> EvmLog { EvmLog::new( Log { @@ -113,19 +153,23 @@ mod tests { #[test] fn parser_requires_block_number() { - let error = parse_log(log(None, Some(0)), test_extractor).unwrap_err(); + let error = parse_log(log(None, Some(0)), &Extractor::Strict(test_extractor)).unwrap_err(); assert!(error.to_string().contains("missing its block number")); } #[test] fn parser_requires_log_index() { - let error = parse_log(log(Some(1), None), test_extractor).unwrap_err(); + let error = parse_log(log(Some(1), None), &Extractor::Strict(test_extractor)).unwrap_err(); assert!(error.to_string().contains("missing its log index")); } #[test] fn parser_rejects_failed_contract_decode() { - let error = parse_log(log(Some(1), Some(0)), rejected_extractor).unwrap_err(); + let error = parse_log( + log(Some(1), Some(0)), + &Extractor::Strict(rejected_extractor), + ) + .unwrap_err(); assert!(error.to_string().contains("could not be decoded")); } @@ -133,7 +177,9 @@ mod tests { fn parser_converts_valid_log() { let source = log(Some(7), Some(3)); let id = source.id; - let event = parse_log(source, test_extractor).unwrap(); + let event = parse_log(source, &Extractor::Strict(test_extractor)) + .unwrap() + .unwrap(); assert_eq!(event.get_id(), id); let (_, _, block) = event.split(); assert_eq!(block, 7); @@ -172,4 +218,23 @@ mod tests { InterfoldEvmEvent::Rejected(EvmLogRejected { id, chain_id: 1, .. }) if id == expected_id )); } + + #[actix::test] + async fn parser_marks_a_benign_skip_as_processed() { + let (tx, mut rx) = mpsc::unbounded_channel(); + let collector = Collector(tx).start().recipient(); + let parser = EvmParser::new_version_aware(&collector, skipped_extractor).start(); + let unsupported = log(Some(7), Some(3)); + let expected_id = unsupported.id; + + parser + .send(InterfoldEvmEvent::Log(unsupported)) + .await + .unwrap(); + + assert_eq!( + rx.recv().await.expect("processed marker"), + InterfoldEvmEvent::Processed(expected_id) + ); + } } diff --git a/crates/evm/src/interfold/events.rs b/crates/evm/src/interfold/events.rs index 7bab07554b..0a068171bc 100644 --- a/crates/evm/src/interfold/events.rs +++ b/crates/evm/src/interfold/events.rs @@ -9,6 +9,7 @@ use crate::contracts::IInterfold; use alloy::primitives::{LogData, B256}; use alloy::sol_types::SolEvent; +use anyhow::{anyhow, Context as _, Result}; use e3_events::E3id; use e3_events::InterfoldEventData; use e3_events::{ @@ -20,7 +21,7 @@ use e3_trbfv::helpers::calculate_error_size; use e3_utils::ArcBytes; use e3_zk_helpers::CiphernodesCommitteeSize; use num_bigint::BigUint; -use tracing::{error, info, trace, warn}; +use tracing::{info, trace, warn}; struct E3RequestedWithChainId(pub IInterfold::E3Requested, pub u64); @@ -296,32 +297,27 @@ fn indexed_u256( topics: &[B256], index: usize, event_type: &str, -) -> Option { +) -> Result { topics .get(index) .map(|topic| alloy::primitives::U256::from_be_bytes(topic.0)) - .or_else(|| { - error!("{event_type} missing indexed topic {index}"); - None - }) + .ok_or_else(|| anyhow!("{event_type} missing indexed topic {index}")) } pub(crate) fn extractor( data: &LogData, topics: &[B256], chain_id: u64, -) -> Option { +) -> Result> { let topic0 = topics.first(); match topic0 { Some(&IInterfold::E3Requested::SIGNATURE_HASH) => { - let Ok(event) = IInterfold::E3Requested::decode_log_data(data) else { - error!("Error parsing event E3Requested after topic matched!"); - return None; - }; + let event = IInterfold::E3Requested::decode_log_data(data) + .context("failed to decode E3Requested after its topic matched")?; match E3RequestedWithChainId(event, chain_id).try_into_e3_requested() { - Ok(payload) => Some(payload.into()), + Ok(payload) => Ok(Some(payload.into())), Err(e) => { - error!( + warn!( chain_id = chain_id, "Skipping E3Requested: this node cannot process it and will NOT participate \ in this E3. This usually indicates a version skew between the ciphernode \ @@ -329,59 +325,51 @@ pub(crate) fn extractor( size). Cause: {}", e ); - None + Ok(None) } } } Some(&IInterfold::CiphertextOutputPublished::SIGNATURE_HASH) => { - let Ok(mut event) = IInterfold::CiphertextOutputPublished::decode_log_data(data) else { - error!("Error parsing event CiphertextOutputPublished after topic matched!"); - return None; - }; + let mut event = IInterfold::CiphertextOutputPublished::decode_log_data(data) + .context("failed to decode CiphertextOutputPublished after its topic matched")?; // e3Id is indexed → extract from topics[1], not log data if let Some(e3_id_topic) = topics.get(1) { event.e3Id = alloy::primitives::U256::from_be_bytes(e3_id_topic.0); } else { - error!("CiphertextOutputPublished missing indexed e3Id in topics!"); - return None; + return Err(anyhow!( + "CiphertextOutputPublished missing indexed e3Id in topics" + )); } - Some(InterfoldEventData::from( + Ok(Some(InterfoldEventData::from( CiphertextOutputPublishedWithChainId(event, chain_id), - )) + ))) } Some(&IInterfold::InputPublished::SIGNATURE_HASH) => { - let Ok(mut event) = IInterfold::InputPublished::decode_log_data(data) else { - error!("Error parsing event InputPublished after topic matched!"); - return None; - }; + let mut event = IInterfold::InputPublished::decode_log_data(data) + .context("failed to decode InputPublished after its topic matched")?; event.e3Id = indexed_u256(topics, 1, "InputPublished")?; - Some(InputPublishedWithChainId(event, chain_id).into()) + Ok(Some(InputPublishedWithChainId(event, chain_id).into())) } Some(&IInterfold::E3Failed::SIGNATURE_HASH) => { - let Ok(mut event) = IInterfold::E3Failed::decode_log_data(data) else { - error!("Error parsing event E3Failed after topic matched!"); - return None; - }; + let mut event = IInterfold::E3Failed::decode_log_data(data) + .context("failed to decode E3Failed after its topic matched")?; event.e3Id = indexed_u256(topics, 1, "E3Failed")?; info!( "E3Failed event received: e3_id={}, stage={:?}, reason={:?}", event.e3Id, event.failedAtStage, event.reason ); - Some(InterfoldEventData::from(E3FailedWithChainId( + Ok(Some(InterfoldEventData::from(E3FailedWithChainId( event, chain_id, - ))) + )))) } Some(&IInterfold::E3StageChanged::SIGNATURE_HASH) => { - let Ok(mut event) = IInterfold::E3StageChanged::decode_log_data(data) else { - error!("Error parsing event E3StageChanged after topic matched!"); - return None; - }; + let mut event = IInterfold::E3StageChanged::decode_log_data(data) + .context("failed to decode E3StageChanged after its topic matched")?; // e3Id is indexed → extract from topics[1], not log data if let Some(e3_id_topic) = topics.get(1) { event.e3Id = alloy::primitives::U256::from_be_bytes(e3_id_topic.0); } else { - error!("E3StageChanged missing indexed e3Id in topics!"); - return None; + return Err(anyhow!("E3StageChanged missing indexed e3Id in topics")); } trace!( "E3StageChanged event received: e3_id={}, {:?} -> {:?}", @@ -389,65 +377,58 @@ pub(crate) fn extractor( event.previousStage, event.newStage ); - Some(InterfoldEventData::from(E3StageChangedWithChainId( + Ok(Some(InterfoldEventData::from(E3StageChangedWithChainId( event, chain_id, - ))) + )))) } Some(&IInterfold::PlaintextOutputPublished::SIGNATURE_HASH) => { - let Ok(mut event) = IInterfold::PlaintextOutputPublished::decode_log_data(data) else { - error!("Error parsing event PlaintextOutputPublished after topic matched!"); - return None; - }; + let mut event = IInterfold::PlaintextOutputPublished::decode_log_data(data) + .context("failed to decode PlaintextOutputPublished after its topic matched")?; // e3Id is indexed → extract from topics[1], not log data if let Some(e3_id_topic) = topics.get(1) { event.e3Id = alloy::primitives::U256::from_be_bytes(e3_id_topic.0); } else { - error!("PlaintextOutputPublished missing indexed e3Id in topics!"); - return None; + return Err(anyhow!( + "PlaintextOutputPublished missing indexed e3Id in topics" + )); } info!( "PlaintextOutputPublished event received: e3_id={}", event.e3Id ); - Some(InterfoldEventData::from( + Ok(Some(InterfoldEventData::from( PlaintextOutputPublishedWithChainId(event, chain_id), - )) + ))) } Some(&IInterfold::RewardsDistributed::SIGNATURE_HASH) => { - let Ok(mut event) = IInterfold::RewardsDistributed::decode_log_data(data) else { - error!("Error parsing event RewardsDistributed after topic matched!"); - return None; - }; + let mut event = IInterfold::RewardsDistributed::decode_log_data(data) + .context("failed to decode RewardsDistributed after its topic matched")?; event.e3Id = indexed_u256(topics, 1, "RewardsDistributed")?; - Some(RewardsDistributedWithChainId(event, chain_id).into()) + Ok(Some(RewardsDistributedWithChainId(event, chain_id).into())) } Some(&IInterfold::RewardCredited::SIGNATURE_HASH) => { - let Ok(mut event) = IInterfold::RewardCredited::decode_log_data(data) else { - error!("Error parsing event RewardCredited after topic matched!"); - return None; - }; + let mut event = IInterfold::RewardCredited::decode_log_data(data) + .context("failed to decode RewardCredited after its topic matched")?; event.e3Id = indexed_u256(topics, 1, "RewardCredited")?; - Some(RewardCreditedWithChainId(event, chain_id).into()) + Ok(Some(RewardCreditedWithChainId(event, chain_id).into())) } Some(&IInterfold::RewardClaimed::SIGNATURE_HASH) => { - let Ok(mut event) = IInterfold::RewardClaimed::decode_log_data(data) else { - error!("Error parsing event RewardClaimed after topic matched!"); - return None; - }; + let mut event = IInterfold::RewardClaimed::decode_log_data(data) + .context("failed to decode RewardClaimed after its topic matched")?; event.e3Id = indexed_u256(topics, 1, "RewardClaimed")?; - Some(RewardClaimedWithChainId(event, chain_id).into()) + Ok(Some(RewardClaimedWithChainId(event, chain_id).into())) } _ => { trace!( topic=?topic0, "Preserving event without a typed Interfold.sol decoder" ); - Some(crate::domain::evm_log_observation::observe( + Ok(Some(crate::domain::evm_log_observation::observe( "Interfold", data, topics, chain_id, - )) + ))) } } } @@ -490,7 +471,8 @@ mod tests { &log_data, &[IInterfold::E3StageChanged::SIGNATURE_HASH, e3_id_topic], 7, - ); + ) + .unwrap(); match out { Some(InterfoldEventData::E3StageChanged(data)) => { assert_eq!(data.previous_stage, E3Stage::Requested); @@ -505,11 +487,11 @@ mod tests { fn test_extractor_preserves_unknown_topic() { let log_data = LogData::default(); assert!(matches!( - extractor(&log_data, &[B256::ZERO], 1), + extractor(&log_data, &[B256::ZERO], 1).unwrap(), Some(InterfoldEventData::EvmLogObserved(_)) )); assert!(matches!( - extractor(&log_data, &[], 1), + extractor(&log_data, &[], 1).unwrap(), Some(InterfoldEventData::EvmLogObserved(_)) )); } @@ -523,7 +505,7 @@ mod tests { index: U256::from(3), }; let input_log = input.encode_log_data(); - let out = extractor(&input_log, input_log.topics(), 10); + let out = extractor(&input_log, input_log.topics(), 10).unwrap(); match out { Some(InterfoldEventData::InputPublished(event)) => { assert_eq!(event.e3_id, E3id::new("8", 10)); @@ -540,7 +522,7 @@ mod tests { amount: U256::from(500), }; let reward_log = reward.encode_log_data(); - let out = extractor(&reward_log, reward_log.topics(), 10); + let out = extractor(&reward_log, reward_log.topics(), 10).unwrap(); match out { Some(InterfoldEventData::RewardCredited(event)) => { assert_eq!(event.e3_id, E3id::new("8", 10)); @@ -559,7 +541,7 @@ mod tests { proof: Bytes::from_static(b"c7"), }; let log = event.encode_log_data(); - let out = extractor(&log, log.topics(), 100); + let out = extractor(&log, log.topics(), 100).unwrap(); match out { Some(InterfoldEventData::PlaintextOutputPublished(event)) => { assert_eq!(event.e3_id, E3id::new("18", 100)); @@ -569,4 +551,42 @@ mod tests { other => panic!("expected PlaintextOutputPublished, got {other:?}"), } } + + #[test] + fn unsupported_e3_requested_is_a_benign_skip() { + let event = IInterfold::E3Requested { + e3Id: U256::from(19), + e3: IInterfold::E3 { + seed: U256::ZERO, + committeeSize: u8::MAX, + requestBlock: U256::ZERO, + inputWindow: [U256::ZERO; 2], + encryptionSchemeId: B256::ZERO, + e3Program: Address::ZERO, + paramSet: 0, + customParams: Bytes::new(), + decryptionVerifier: Address::ZERO, + pkVerifier: Address::ZERO, + committeePublicKey: B256::ZERO, + ciphertextOutput: B256::ZERO, + plaintextOutput: Bytes::new(), + requester: Address::ZERO, + proofAggregationEnabled: false, + }, + e3Program: Address::ZERO, + }; + let log = event.encode_log_data(); + + assert!(extractor(&log, log.topics(), 100).unwrap().is_none()); + } + + #[test] + fn malformed_e3_requested_is_rejected() { + let malformed = LogData::new_unchecked( + vec![IInterfold::E3Requested::SIGNATURE_HASH], + Default::default(), + ); + + assert!(extractor(&malformed, malformed.topics(), 100).is_err()); + } } diff --git a/crates/evm/src/interfold/reader.rs b/crates/evm/src/interfold/reader.rs index 7d2bcedae5..e8022ef791 100644 --- a/crates/evm/src/interfold/reader.rs +++ b/crates/evm/src/interfold/reader.rs @@ -14,6 +14,6 @@ pub struct InterfoldSolReader; impl InterfoldSolReader { pub fn setup(next: &EvmEventProcessor) -> Addr { - EvmParser::new(next, extractor).start() + EvmParser::new_version_aware(next, extractor).start() } } From e67d9509d30f3dabfb19d814d2686c57db6c27cc Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 14 Jul 2026 22:31:57 +0500 Subject: [PATCH 58/62] fix(net): keep live ingress running after lag --- agent/CRATES_ARCHITECTURE.md | 3 +- agent/flow-trace/00_INDEX.md | 1 + .../06_DEACTIVATION_AND_COMPLETION.md | 3 + crates/net/src/event_buffer/actor.rs | 1 + crates/net/src/event_buffer/handlers.rs | 56 ++++++++++++++++++- 5 files changed, 61 insertions(+), 3 deletions(-) diff --git a/agent/CRATES_ARCHITECTURE.md b/agent/CRATES_ARCHITECTURE.md index 4382c83251..f66e42def7 100644 --- a/agent/CRATES_ARCHITECTURE.md +++ b/agent/CRATES_ARCHITECTURE.md @@ -433,7 +433,8 @@ transport channels. Gossipsub and direct-request/DHT decoding have explicit byte limits. Translation actors accept only the protocol event allowlist before publishing remote events, and their broadcast-to-actor ingress loops await mailbox acceptance and stop when the destination actor closes. Startup buffering is bounded by both event -count and estimated bytes and fails readiness on overflow or broadcast lag. Historical +count and estimated bytes and fails readiness on overflow or broadcast lag; after +`SyncEnded`, broadcast lag is warned and skipped without stopping the ingress loop. Historical direct sync requires advancing cursors and enforces one cumulative page, event, byte, and time budget across all aggregate fetches and recovery retries in a startup attempt. diff --git a/agent/flow-trace/00_INDEX.md b/agent/flow-trace/00_INDEX.md index 335bc90352..09d401290b 100644 --- a/agent/flow-trace/00_INDEX.md +++ b/agent/flow-trace/00_INDEX.md @@ -199,3 +199,4 @@ _Found during source-code cross-referencing of these trace documents._ | 11 | **Large EventStore replay startup failure** | Resolved | Restart replay now awaits each EventBus handler result before submitting the next persisted event. A backlog larger than the 2,560-entry EventBus mailbox no longer fails startup with `SendError::Full`; progress is logged every 10,000 EventBus-handled events. The EventBus still fans out with unacknowledged `do_send`, and replay still materializes the full range in memory, so end-to-end queue and memory bounds remain open. | | 12 | **Canonical DKG H/N aggregation shape** | Resolved | The Rust DKG aggregation path no longer asserts that honest proof count `H` equals full committee size `N`. Every supported preset has `H < N`; validation now requires exactly `H` NodeFold proofs with unique in-range party IDs and exactly `N` ordered committee addresses, matching the producer and compiled Noir witness. The removed assertion affected debug/test binaries; circuit semantics and public inputs are unchanged. | | 13 | **Forward-compatible E3 version skew** | Resolved | A well-formed `E3Requested` carrying an unsupported committee-size or BFV-preset enum is now classified as a benign skip for that older binary. The parser emits an internal `Processed` marker so historical ordering and startup can advance without dispatching the E3. ABI decode failures and missing indexed topics remain rejected and fail the chain pipeline closed. | +| 14 | **Post-startup network lag recovery** | Resolved | `NetEventBuffer` still fails readiness closed if its broadcast input lags during startup buffering, because historical/live reconciliation would be incomplete. Once `SyncEnded` has moved it to `Running`, a `Lagged(n)` signal is warned and skipped, and the ingress receive loop continues from the oldest retained event instead of stopping the buffer and permanently deafening gossip consumers. | diff --git a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md index 304b6c5e7b..0cd380a38d 100644 --- a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md +++ b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md @@ -262,6 +262,9 @@ overload signal: they emit a bounded structured warning containing only the stat skipped-event count, then continue from the oldest retained event. Only channel closure ends a receive task. A lag can still drop the reported `n` events, but a single burst no longer permanently disables gossip translation, document notifications, or historical-sync/readiness handling. +`NetEventBuffer` applies the same continue policy only after `SyncEnded`; lag during its startup +buffering window remains a fail-closed readiness error because those skipped events cannot yet be +reconciled safely. EventStore replay uses a disk-backed external merge: per-aggregate pages are sorted into secure temporary runs, then compacted and merged with bounded file-descriptor fan-in. Replay waits for diff --git a/crates/net/src/event_buffer/actor.rs b/crates/net/src/event_buffer/actor.rs index f7318de3b1..8bb0434547 100644 --- a/crates/net/src/event_buffer/actor.rs +++ b/crates/net/src/event_buffer/actor.rs @@ -13,6 +13,7 @@ use e3_events::{ use e3_utils::MAILBOX_LIMIT; use tokio::sync::broadcast::{self, error::RecvError}; use tokio::sync::oneshot; +use tracing::warn; use crate::domain::net_buffer::{BufferDecision, NetEventBufferState}; use crate::events::NetEvent; diff --git a/crates/net/src/event_buffer/handlers.rs b/crates/net/src/event_buffer/handlers.rs index db420ba6b1..cb2b31ba27 100644 --- a/crates/net/src/event_buffer/handlers.rs +++ b/crates/net/src/event_buffer/handlers.rs @@ -29,8 +29,9 @@ impl Actor for NetEventBuffer { } } Err(RecvError::Lagged(skipped)) => { - let _ = addr.send(NetInputLagged(skipped)).await; - break; + if addr.send(NetInputLagged(skipped)).await.is_err() { + break; + } } Err(RecvError::Closed) => break, } @@ -71,6 +72,13 @@ impl Handler for NetEventBuffer { type Result = (); fn handle(&mut self, msg: NetInputLagged, ctx: &mut Self::Context) { + if self.state.is_running() { + warn!( + skipped_events = msg.0, + "Network event buffer input lagged after startup; continuing from the oldest retained event" + ); + return; + } self.fail_closed( anyhow!( "network event input skipped {} events because its bounded broadcast receiver lagged", @@ -90,3 +98,47 @@ impl Handler for NetEventBuffer { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::events::GossipData; + use e3_ciphernode_builder::EventSystem; + use std::time::Duration; + + #[actix::test] + async fn lag_after_startup_keeps_the_buffer_actor_live() -> Result<()> { + let system = EventSystem::new().with_fresh_bus(); + let bus = system.handle()?.enable("post-startup-lag"); + let (_input_tx, input_rx) = broadcast::channel(1); + let (output_tx, mut output_rx) = broadcast::channel(8); + let (readiness, _readiness_rx) = oneshot::channel(); + let mut state = NetEventBufferState::syncing(); + state.run()?; + let actor = NetEventBuffer { + state, + input_rx: Some(input_rx), + output_tx, + bus, + max_events: 8, + max_bytes: 1_024, + readiness: Some(readiness), + } + .start(); + + actor.send(NetInputLagged(4)).await?; + actor + .send(IncomingNetEvent(NetEvent::GossipData( + GossipData::GossipBytes(vec![7]), + ))) + .await?; + + let forwarded = tokio::time::timeout(Duration::from_secs(1), output_rx.recv()).await??; + assert!(matches!( + forwarded, + NetEvent::GossipData(GossipData::GossipBytes(bytes)) if bytes == vec![7] + )); + assert!(actor.connected()); + Ok(()) + } +} From 5b552a29714194fee73b89569af67e912a2a05df Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 14 Jul 2026 22:33:14 +0500 Subject: [PATCH 59/62] fix(events): dedupe redelivery by stable identity --- agent/CRATES_ARCHITECTURE.md | 3 ++ agent/flow-trace/00_INDEX.md | 1 + .../06_DEACTIVATION_AND_COMPLETION.md | 3 ++ crates/events/src/eventstore.rs | 40 ++++++++++++++++--- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/agent/CRATES_ARCHITECTURE.md b/agent/CRATES_ARCHITECTURE.md index f66e42def7..333b85f23d 100644 --- a/agent/CRATES_ARCHITECTURE.md +++ b/agent/CRATES_ARCHITECTURE.md @@ -367,6 +367,9 @@ The append-only event log is the durable source of truth. The timestamp index an snapshots are derived state. EventStore construction reconciles missing index rows from the log in strict 1,024-record pages and fail-stops on a gap or unusable index; it does not decode the complete log solely to rebuild the index. +Timestamp admission deduplicates by stable event ID plus payload, so the same logical event may +return through historical network sync with a different transport source without colliding. A +different payload at an already-indexed HLC timestamp remains an integrity failure. Post-snapshot events are queried per aggregate in 1,024-event pages and sorted into secure temporary runs. Runs are compacted with bounded fan-in and merged globally by persisted HLC timestamp, so memory and open-file use do not scale with the entire diff --git a/agent/flow-trace/00_INDEX.md b/agent/flow-trace/00_INDEX.md index 09d401290b..ff03708014 100644 --- a/agent/flow-trace/00_INDEX.md +++ b/agent/flow-trace/00_INDEX.md @@ -200,3 +200,4 @@ _Found during source-code cross-referencing of these trace documents._ | 12 | **Canonical DKG H/N aggregation shape** | Resolved | The Rust DKG aggregation path no longer asserts that honest proof count `H` equals full committee size `N`. Every supported preset has `H < N`; validation now requires exactly `H` NodeFold proofs with unique in-range party IDs and exactly `N` ordered committee addresses, matching the producer and compiled Noir witness. The removed assertion affected debug/test binaries; circuit semantics and public inputs are unchanged. | | 13 | **Forward-compatible E3 version skew** | Resolved | A well-formed `E3Requested` carrying an unsupported committee-size or BFV-preset enum is now classified as a benign skip for that older binary. The parser emits an internal `Processed` marker so historical ordering and startup can advance without dispatching the E3. ABI decode failures and missing indexed topics remain rejected and fail the chain pipeline closed. | | 14 | **Post-startup network lag recovery** | Resolved | `NetEventBuffer` still fails readiness closed if its broadcast input lags during startup buffering, because historical/live reconciliation would be incomplete. Once `SyncEnded` has moved it to `Running`, a `Lagged(n)` signal is warned and skipped, and the ingress receive loop continues from the oldest retained event instead of stopping the buffer and permanently deafening gossip consumers. | +| 15 | **Local/network event redelivery collision** | Resolved | EventStore now treats the same HLC timestamp plus stable event ID and equal payload as an idempotent duplicate even when transport context differs (`Local` versus `Net`). It retains the first stored context and still fails closed when different payloads claim the same timestamp, preserving collision detection without panicking on sync/resync redelivery. | diff --git a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md index 0cd380a38d..4cb5fbc88a 100644 --- a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md +++ b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md @@ -242,6 +242,9 @@ On restart: │ 5. Historical libp2p sync retries failed aggregate fetches after reconnects │ and also on bounded retry intervals even without a new connection event │ 6. Sort & publish merged events by HLC timestamp +│ → A logical event returned by a peer with its source changed from Local to Net is +│ idempotent when timestamp, stable event ID, and payload match the stored record; +│ a different payload at the same timestamp still fails closed as a collision │ → ComputeEffectGate has already subscribed and buffers ComputeRequest │ effects, deduplicating semantic retries while replay is in progress │ 7. Enable effects (writers may submit only after this point) diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index 25f80b5297..f43e7334ae 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -22,7 +22,9 @@ pub struct EventStore { impl EventStore { /// Attempt to store an event. Returns the sequenced event on success, - /// `None` if the complete event is an exact duplicate, or an error on failure. + /// `None` if the event has the same stable identity and payload as the stored event, or an + /// error on failure. Transport context such as `Local` versus `Net` is not part of the + /// logical event identity and may legitimately change during historical redelivery. pub fn store_event( &mut self, event: InterfoldEvent, @@ -42,13 +44,15 @@ impl EventStore { log returned sequence {logged_seq}" ); } - if existing == event { + if existing.id() == event.id() && existing.get_data() == event.get_data() { warn!( timestamp = ts, sequence = indexed_seq, event_id = %event.id(), event_type = %event.get_data().event_type(), - "Ignoring exact duplicate event" + incoming_source = ?event.source(), + stored_source = ?existing.source(), + "Ignoring duplicate logical event" ); return Ok(None); } @@ -487,6 +491,16 @@ mod tests { ) } + fn make_distinct_event(ts: u128, source: EventSource) -> InterfoldEvent { + InterfoldEvent::::new_with_timestamp( + TestEvent::new("different", 2).into(), + None, + ts, + None, + source, + ) + } + fn make_local_event(ts: u128) -> InterfoldEvent { make_event(ts, EventSource::Local) } @@ -672,11 +686,27 @@ mod tests { } #[test] - fn conflicting_event_at_same_timestamp_fails_immediately() { + fn local_event_redelivered_from_network_is_idempotent() { + let mut store = new_store(); + store.store_event(make_local_event(100)).unwrap(); + + assert!(store + .store_event(make_network_event(100)) + .unwrap() + .is_none()); + let stored = store.log.read_from(1).next().unwrap().1; + assert_eq!(stored.source(), EventSource::Local); + assert_eq!(store.log.read_from(1).count(), 1); + } + + #[test] + fn conflicting_payload_at_same_timestamp_fails_immediately() { let mut store = new_store(); store.store_event(make_local_event(100)).unwrap(); - let error = store.store_event(make_network_event(100)).unwrap_err(); + let error = store + .store_event(make_distinct_event(100, EventSource::Net)) + .unwrap_err(); let message = error.to_string(); assert!(message.contains("timestamp collision"), "{message}"); assert!(message.contains("sequence 1"), "{message}"); From c13e9280d9d73c290bb3dfa1c8282619b8d2d9fd Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 14 Jul 2026 22:42:46 +0500 Subject: [PATCH 60/62] fix(cli): keep provisioning secrets out of argv --- agent/flow-trace/00_INDEX.md | 1 + agent/flow-trace/01_REGISTRATION.md | 5 ++ crates/ciphernode-entrypoint.sh | 32 +++++------ crates/cli/src/ciphernode/mod.rs | 32 ++++++++--- crates/cli/src/ciphernode/setup.rs | 16 +++++- crates/cli/src/cli.rs | 6 ++ crates/cli/src/helpers/mod.rs | 57 ++++++++++++++++++- crates/cli/src/password.rs | 20 ++++++- crates/cli/src/wallet.rs | 28 +++++++++- dappnode/Dockerfile | 2 + dappnode/README.md | 7 ++- dappnode/entrypoint.sh | 28 ++++------ dappnode/provision-credentials.exp | 87 +++++++++++++++++++++++++++++ dappnode/tests/test-hardening.sh | 49 ++++++++++++---- 14 files changed, 308 insertions(+), 62 deletions(-) create mode 100644 dappnode/provision-credentials.exp diff --git a/agent/flow-trace/00_INDEX.md b/agent/flow-trace/00_INDEX.md index ff03708014..907bed04ab 100644 --- a/agent/flow-trace/00_INDEX.md +++ b/agent/flow-trace/00_INDEX.md @@ -18,6 +18,7 @@ ``` 1. SETUP interfold ciphernode setup → Config, password, private key stored locally + → Automation passes secrets over stdin, never argv/environment metadata 2. BOND interfold ciphernode license bond --amount N → FOLD tokens locked in BondingRegistry diff --git a/agent/flow-trace/01_REGISTRATION.md b/agent/flow-trace/01_REGISTRATION.md index 756f8f0ae4..4f2c9c89dc 100644 --- a/agent/flow-trace/01_REGISTRATION.md +++ b/agent/flow-trace/01_REGISTRATION.md @@ -6,6 +6,11 @@ A ciphernode operator goes from zero to registered on-chain through a series of configure local state, encrypt credentials, and submit an on-chain transaction to the `BondingRegistry`. +For non-interactive provisioning, `password set`, `wallet set`, and `ciphernode setup` expose +`--password-stdin` / `--private-key-stdin` alternatives. Container entrypoints use these stdin or +hidden-prompt paths so encryption passwords and private keys do not appear in process arguments or +environment metadata. + --- ## Step 1: `interfold ciphernode setup` diff --git a/crates/ciphernode-entrypoint.sh b/crates/ciphernode-entrypoint.sh index 3855e56b1c..30598a8dde 100644 --- a/crates/ciphernode-entrypoint.sh +++ b/crates/ciphernode-entrypoint.sh @@ -1,5 +1,7 @@ #!/bin/bash -set -e +set -Eeuo pipefail + +umask 077 # Paths to config and secrets CONFIG_FILE="$CONFIG_DIR/config.yaml" @@ -16,28 +18,26 @@ if [ ! -f "$SECRETS_FILE" ]; then exit 1 fi -# Read secrets from the JSON file -PRIVATE_KEY=$(jq -r '.private_key' "$SECRETS_FILE") -PASSWORD=$(jq -r '.password' "$SECRETS_FILE") -NETWORK_PRIVATE_KEY=$(jq -r '.network_private_key' "$SECRETS_FILE") - -if [ -z "$PRIVATE_KEY" ] || [ -z "$PASSWORD" ] || [ -z "$NETWORK_PRIVATE_KEY" ]; then - echo "Error: Missing 'private_key', 'password' or 'network_private_key' in secrets file!" +jq -e ' + type == "object" and + (.password | type == "string" and length > 0) and + (.private_key | type == "string" and test("^0x[0-9a-fA-F]{64}$")) +' "$SECRETS_FILE" >/dev/null || { + echo "Error: Invalid 'password' or 'private_key' in secrets file!" exit 1 -fi +} # Set password echo "Setting password" -interfold password set --config "$CONFIG_FILE" --password "$PASSWORD" - -# Set network private key -echo "Setting network private key" -interfold net set-key --config "$CONFIG_FILE" --net-keypair "$NETWORK_PRIVATE_KEY" +jq -er '.password' "$SECRETS_FILE" | interfold password set --config "$CONFIG_FILE" --password-stdin echo "Setting wallet key" -interfold wallet set --config "$CONFIG_FILE" --private-key "$PRIVATE_KEY" +# The refactored wallet command atomically derives and stores the libp2p key +# from the operator key, so a separate network-key command is no longer used. +jq -er '.private_key' "$SECRETS_FILE" | interfold wallet set --config "$CONFIG_FILE" --private-key-stdin + +rm -f "$SECRETS_FILE" echo "Starting ciphernode" exec interfold start -v --config "$CONFIG_FILE" - diff --git a/crates/cli/src/ciphernode/mod.rs b/crates/cli/src/ciphernode/mod.rs index 4c96838dae..e177632fad 100644 --- a/crates/cli/src/ciphernode/mod.rs +++ b/crates/cli/src/ciphernode/mod.rs @@ -43,13 +43,31 @@ pub enum CiphernodeCommands { #[arg(long = "rpc-url", short = 'r')] rpc_url: Option, - /// The password - #[arg(short='p', long, value_parser = parse_zeroizing)] - password: Option>, - - /// Wallet Private Key - #[arg(short='k', long, value_parser = ensure_hex_zeroizing)] - private_key: Option>, + /// The password + #[arg( + short = 'p', + long, + value_parser = parse_zeroizing, + conflicts_with = "password_stdin" + )] + password: Option>, + + /// Read the password from the first requested line on stdin + #[arg(long, conflicts_with = "password")] + password_stdin: bool, + + /// Wallet Private Key + #[arg( + short = 'k', + long, + value_parser = ensure_hex_zeroizing, + conflicts_with = "private_key_stdin" + )] + private_key: Option>, + + /// Read the private key from the next requested line on stdin + #[arg(long, conflicts_with = "private_key")] + private_key_stdin: bool, }, /// Manage FOLD license tokens and bonding state License { diff --git a/crates/cli/src/ciphernode/setup.rs b/crates/cli/src/ciphernode/setup.rs index 0b01ca5fee..39bf85732a 100644 --- a/crates/cli/src/ciphernode/setup.rs +++ b/crates/cli/src/ciphernode/setup.rs @@ -15,6 +15,7 @@ use std::path::PathBuf; use tracing::instrument; use zeroize::Zeroizing; +use crate::helpers::read_secret_line; use crate::password_set::ask_for_password; use crate::wallet_set::ask_for_private_key; @@ -22,9 +23,20 @@ use crate::wallet_set::ask_for_private_key; pub async fn execute( out: Console, rpc_url: Option, - password: Option>, - private_key: Option>, + mut password: Option>, + password_stdin: bool, + mut private_key: Option>, + private_key_stdin: bool, ) -> Result<()> { + if password_stdin || private_key_stdin { + let mut stdin = std::io::stdin().lock(); + if password_stdin { + password = Some(read_secret_line(&mut stdin, "password")?); + } + if private_key_stdin { + private_key = Some(read_secret_line(&mut stdin, "private key")?); + } + } let pw = ask_for_password(password)?; let rpc_url = match rpc_url { Some(url) => { diff --git a/crates/cli/src/cli.rs b/crates/cli/src/cli.rs index 5fb2317267..ba1e9d3035 100644 --- a/crates/cli/src/cli.rs +++ b/crates/cli/src/cli.rs @@ -109,14 +109,18 @@ impl Cli { command: CiphernodeCommands::Setup { rpc_url, password, + password_stdin, private_key, + private_key_stdin, } } => { ciphernode::setup::execute( out, rpc_url, password, + password_stdin, private_key, + private_key_stdin, ) .await?; } @@ -126,7 +130,9 @@ impl Cli { out, None, None, + false, None, + false, ) .await?; }, diff --git a/crates/cli/src/helpers/mod.rs b/crates/cli/src/helpers/mod.rs index 864f99b0a4..f13b40f003 100644 --- a/crates/cli/src/helpers/mod.rs +++ b/crates/cli/src/helpers/mod.rs @@ -4,7 +4,8 @@ // without even the implied warranty of MERCHANTABILITY // or FITNESS FOR A PARTICULAR PURPOSE. -use anyhow::{bail, Result}; +use anyhow::{bail, Context, Result}; +use std::io::BufRead; use zeroize::{Zeroize, Zeroizing}; pub mod compile_id; @@ -21,6 +22,35 @@ pub fn ensure_hex_zeroizing(s: &str) -> Result> { parse_zeroizing(ensure_hex(s)?) } +/// Read one secret from stdin without placing it in argv or the environment. +/// +/// Only the line terminator is removed. Other whitespace remains part of the +/// secret so callers can apply their normal validation rules. +pub fn read_secret_line(reader: &mut impl BufRead, description: &str) -> Result> { + let mut value = Zeroizing::new(String::new()); + let bytes_read = reader + .read_line(&mut value) + .with_context(|| format!("failed to read {description} from stdin"))?; + if bytes_read == 0 { + bail!("{description} was not provided on stdin"); + } + + if value.ends_with('\n') { + value.pop(); + if value.ends_with('\r') { + value.pop(); + } + } + if value.contains(['\r', '\n', '\0']) { + bail!("{description} on stdin must be a single line"); + } + if value.is_empty() { + bail!("{description} on stdin must not be empty"); + } + + Ok(value) +} + /// Ensure a hexadecimal number fn ensure_hex(s: &str) -> Result<&str> { if !s.starts_with("0x") { @@ -32,3 +62,28 @@ fn ensure_hex(s: &str) -> Result<&str> { hex::decode(&s[2..])?.zeroize(); Ok(s) } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn secret_line_removes_only_the_line_terminator() -> Result<()> { + let mut input = Cursor::new(b" secret value \r\nnext\n"); + + assert_eq!( + &*read_secret_line(&mut input, "password")?, + " secret value " + ); + assert_eq!(&*read_secret_line(&mut input, "private key")?, "next"); + Ok(()) + } + + #[test] + fn secret_line_rejects_missing_and_embedded_newlines() { + assert!(read_secret_line(&mut Cursor::new(Vec::::new()), "password").is_err()); + assert!(read_secret_line(&mut Cursor::new(b"\n"), "password").is_err()); + assert!(read_secret_line(&mut Cursor::new(b"bad\rvalue\n"), "password").is_err()); + } +} diff --git a/crates/cli/src/password.rs b/crates/cli/src/password.rs index 78a42bd7d6..a335d43d62 100644 --- a/crates/cli/src/password.rs +++ b/crates/cli/src/password.rs @@ -10,15 +10,19 @@ use e3_config::AppConfig; use e3_console::Console; use zeroize::Zeroizing; -use crate::{helpers::parse_zeroizing, password_delete, password_set}; +use crate::{helpers::parse_zeroizing, helpers::read_secret_line, password_delete, password_set}; #[derive(Subcommand, Clone, Debug)] pub enum PasswordCommands { /// Set (or overwrite) a password Set { /// The new password - #[arg(short, long, value_parser = parse_zeroizing)] + #[arg(short, long, value_parser = parse_zeroizing, conflicts_with = "password_stdin")] password: Option>, + + /// Read the new password from one line on stdin + #[arg(long, conflicts_with = "password")] + password_stdin: bool, }, /// Delete the current password @@ -27,7 +31,17 @@ pub enum PasswordCommands { pub async fn execute(out: Console, command: PasswordCommands, config: &AppConfig) -> Result<()> { match command { - PasswordCommands::Set { password } => password_set::execute(out, config, password).await?, + PasswordCommands::Set { + password, + password_stdin, + } => { + let password = if password_stdin { + Some(read_secret_line(&mut std::io::stdin().lock(), "password")?) + } else { + password + }; + password_set::execute(out, config, password).await? + } PasswordCommands::Delete => password_delete::execute(&out, config).await?, }; diff --git a/crates/cli/src/wallet.rs b/crates/cli/src/wallet.rs index 63f9bd15aa..02a822fb3d 100644 --- a/crates/cli/src/wallet.rs +++ b/crates/cli/src/wallet.rs @@ -10,7 +10,10 @@ use e3_config::AppConfig; use e3_console::Console; use zeroize::Zeroizing; -use crate::{helpers::ensure_hex_zeroizing, wallet_get, wallet_set}; +use crate::{ + helpers::{ensure_hex_zeroizing, read_secret_line}, + wallet_get, wallet_set, +}; #[derive(Subcommand, Clone, Debug)] pub enum WalletCommands { @@ -18,8 +21,16 @@ pub enum WalletCommands { Set { /// The private key - note we are leaving as hex string as it is easier to manage with /// the allow Signer coercion - #[arg(long = "private-key", value_parser = ensure_hex_zeroizing)] + #[arg( + long = "private-key", + value_parser = ensure_hex_zeroizing, + conflicts_with = "private_key_stdin" + )] private_key: Option>, + + /// Read the private key from one line on stdin + #[arg(long, conflicts_with = "private_key")] + private_key_stdin: bool, }, /// Get your wallet address Get, @@ -27,7 +38,18 @@ pub enum WalletCommands { pub async fn execute(out: Console, command: WalletCommands, config: AppConfig) -> Result<()> { match command { - WalletCommands::Set { private_key } => { + WalletCommands::Set { + private_key, + private_key_stdin, + } => { + let private_key = if private_key_stdin { + Some(read_secret_line( + &mut std::io::stdin().lock(), + "private key", + )?) + } else { + private_key + }; wallet_set::execute(out, &config, private_key).await? } WalletCommands::Get => wallet_get::execute(out, &config).await?, diff --git a/dappnode/Dockerfile b/dappnode/Dockerfile index 63ed2adfbd..569b2c2d05 100644 --- a/dappnode/Dockerfile +++ b/dappnode/Dockerfile @@ -13,6 +13,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ gettext-base \ iproute2 \ jq \ + expect \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* @@ -31,6 +32,7 @@ RUN chmod +x /usr/local/bin/interfold COPY --chmod=755 entrypoint.sh /usr/local/bin/entrypoint.sh COPY --chmod=755 healthcheck.sh /usr/local/bin/healthcheck.sh COPY --chmod=644 config.template.yaml /opt/config.template.yaml +COPY --chmod=444 provision-credentials.exp /opt/provision-credentials.exp # Labels LABEL maintainer="Gnosis Guild " diff --git a/dappnode/README.md b/dappnode/README.md index 0b9c2ac747..8aece563a1 100644 --- a/dappnode/README.md +++ b/dappnode/README.md @@ -170,7 +170,9 @@ Upload it in the setup wizard as **Ciphernode Credentials JSON**. DAppNode copie of 16 KiB, required fields, and key encodings, then runs the exact commands supported by the pinned Interfold v0.1.8 image. Any failed command aborts startup. Wallet and network keys are encrypted in `/data`; v0.1.8 stores its password key there as a mode-`0400` file. After successful persistence, -the entrypoint removes the combined plaintext upload. +the entrypoint removes the combined plaintext upload. Provisioning sends the secrets through the +CLI's hidden TTY prompts over stdin; plaintext credentials are never placed in process arguments or +container environment variables. The Ethereum key must correspond to `NODE_ADDRESS`. Keep an encrypted offline backup of this JSON; do not paste its contents into package environment variables, `EXTRA_OPTS`, logs, or support @@ -207,7 +209,8 @@ At container startup, `entrypoint.sh`: - contract addresses and deploy blocks 4. Validates the uploaded credential file and programs password, network key, and wallet key. The - pinned v0.1.8 network-key command is `interfold net keypair set`; failures stop startup. + pinned v0.1.8 network-key command is `interfold net keypair set`; an isolated Expect helper feeds + its hidden prompts from stdin and failures stop startup. 5. Builds CLI args, including verbosity and `--peer` flags from `PEERS`. 6. Executes: diff --git a/dappnode/entrypoint.sh b/dappnode/entrypoint.sh index 192a43c0ed..86fadb3caf 100644 --- a/dappnode/entrypoint.sh +++ b/dappnode/entrypoint.sh @@ -8,6 +8,7 @@ CONFIG_DIR="${CONFIG_DIR:-/data}" CONFIG_FILE="${CONFIG_FILE:-$CONFIG_DIR/config.yaml}" TEMPLATE_FILE="${TEMPLATE_FILE:-/opt/config.template.yaml}" SECRETS_FILE="${SECRETS_FILE:-/run/secrets/secrets.json}" +CREDENTIAL_PROVISIONER="${CREDENTIAL_PROVISIONER:-/opt/provision-credentials.exp}" # Interfold v0.1.8 resolves a relative `key_file: key` beside a discovered # /data/config.yaml to this path for the default node profile. PASSWORD_FILE="${PASSWORD_FILE:-$CONFIG_DIR/.enclave/config/_default/key}" @@ -81,29 +82,22 @@ validate_persisted_password_file() { configure_credentials() { validate_secret_file - local password private_key network_private_key - password=$(jq -er '.password' "$SECRETS_FILE") - private_key=$(jq -er '.private_key' "$SECRETS_FILE") - network_private_key=$(jq -er '.network_private_key' "$SECRETS_FILE") + [ -r "$CREDENTIAL_PROVISIONER" ] || fail "credential provisioner is not readable: $CREDENTIAL_PROVISIONER" + + local provisioning_mode=new if [ -e "$PASSWORD_FILE" ]; then validate_persisted_password_file - printf '%s' "$password" | cmp -s - "$PASSWORD_FILE" || fail "uploaded password does not match the persisted credential key" + jq -er '.password' "$SECRETS_FILE" | tr -d '\n' | cmp -s - "$PASSWORD_FILE" \ + || fail "uploaded password does not match the persisted credential key" + provisioning_mode=existing log "Using the matching persisted encryption password." - else - log "Setting encryption password..." - interfold password set --config "$CONFIG_FILE" --password "$password" fi - # v0.1.8 nests this operation under `net keypair`; `net set-key` is not a - # valid command for the image pinned by this package. - log "Setting network key..." - interfold net keypair set --config "$CONFIG_FILE" --net-keypair "$network_private_key" - - log "Setting wallet key..." - interfold wallet set --config "$CONFIG_FILE" --private-key "$private_key" - - unset password private_key network_private_key + log "Provisioning encrypted credentials through hidden stdin prompts..." + jq -jr '[.password, .network_private_key, .private_key][] | @base64 + "\n"' "$SECRETS_FILE" \ + | expect "$CREDENTIAL_PROVISIONER" "$CONFIG_FILE" "$provisioning_mode" \ + || fail "one or more credential commands failed" # DAppNode copies fileUpload content into this container before startup. # Wallet/network keys are encrypted in /data and v0.1.8 stores the password diff --git a/dappnode/provision-credentials.exp b/dappnode/provision-credentials.exp new file mode 100644 index 0000000000..75aa6fcaa6 --- /dev/null +++ b/dappnode/provision-credentials.exp @@ -0,0 +1,87 @@ +#!/usr/bin/expect -f +# Feed credentials to the hidden interactive prompts exposed by Interfold +# v0.1.8. Secrets arrive as base64-encoded lines on stdin; neither the +# plaintext nor the encoded value is placed in argv or the environment. + +set timeout 30 + +proc fail {message {code 1}} { + puts stderr "credential provisioning failed: $message" + exit $code +} + +proc read_secret {description} { + if {[gets stdin encoded] < 0 || $encoded eq ""} { + fail "$description was not provided on stdin" + } + if {[catch {binary decode base64 $encoded} decoded]} { + fail "$description was not valid base64" + } + return $decoded +} + +proc finish_child {description} { + expect { + eof {} + timeout { fail "$description timed out" 124 } + } + set status [wait] + if {[lindex $status 2] != 0} { + fail "$description could not be reaped" + } + set exit_code [lindex $status 3] + if {$exit_code != 0} { + fail "$description exited with status $exit_code" $exit_code + } +} + +proc fail_on_early_exit {description} { + set status [wait] + if {[lindex $status 2] == 0} { + set exit_code [lindex $status 3] + if {$exit_code == 0} { + set exit_code 1 + } + fail "$description exited before prompting (status $exit_code)" $exit_code + } + fail "$description exited before prompting" +} + +proc send_at_prompt {pattern secret description} { + expect { + -re $pattern { send -- "$secret\r" } + timeout { fail "timed out waiting for $description prompt" 124 } + eof { fail_on_early_exit "$description command" } + } +} + +if {$argc != 2} { + fail "expected CONFIG_FILE and provisioning mode" +} +set config_file [lindex $argv 0] +set mode [lindex $argv 1] +if {$mode ne "new" && $mode ne "existing"} { + fail "provisioning mode must be new or existing" +} + +set password [read_secret "password"] +set network_private_key [read_secret "network private key"] +set private_key [read_secret "wallet private key"] + +if {$mode eq "new"} { + spawn -noecho interfold password set --config $config_file + send_at_prompt {Please enter a new password} $password "new password" + send_at_prompt {Please confirm your password} $password "password confirmation" + finish_child "password command" +} +unset password + +spawn -noecho interfold net keypair set --config $config_file +send_at_prompt {Enter your network private key} $network_private_key "network private key" +finish_child "network key command" +unset network_private_key + +spawn -noecho interfold wallet set --config $config_file +send_at_prompt {Enter your Ethereum private key} $private_key "wallet private key" +finish_child "wallet command" +unset private_key diff --git a/dappnode/tests/test-hardening.sh b/dappnode/tests/test-hardening.sh index 08c615bed2..497767c345 100644 --- a/dappnode/tests/test-hardening.sh +++ b/dappnode/tests/test-hardening.sh @@ -14,11 +14,11 @@ grep -Fq 'stop_grace_period: 45s' "$ROOT_DIR/docker-compose.yml" \ || fail "Docker stop grace period must exceed the node shutdown deadline" assert_contains() { - grep -Fq "$2" "$1" || fail "expected '$2' in $1" + grep -Fq -- "$2" "$1" || fail "expected '$2' in $1" } assert_not_contains() { - if grep -Fq "$2" "$1"; then + if grep -Fq -- "$2" "$1"; then fail "did not expect '$2' in $1" fi } @@ -37,20 +37,34 @@ make_mock_interfold() { ' *) operation=unexpected ;;' \ 'esac' \ 'printf "%s\n" "$operation" >> "$CALL_LOG"' \ + 'tr "\0" " " < /proc/$$/cmdline >> "$ARGV_LOG"' \ + 'printf "\n" >> "$ARGV_LOG"' \ '[ "${FAIL_ON:-}" != "$operation" ] || exit 42' \ - 'if [ "$operation" = password ]; then' \ - ' password=' \ - ' while [ "$#" -gt 0 ]; do' \ - ' if [ "$1" = --password ]; then password=$2; break; fi' \ - ' shift' \ - ' done' \ + '[ "$operation" != unexpected ]' \ + > "$bin_dir/interfold" + chmod +x "$bin_dir/interfold" +} + +make_mock_expect() { + local bin_dir=$1 + printf '%s\n' \ + '#!/bin/bash' \ + 'set -Eeuo pipefail' \ + 'read -r password_b64' \ + 'read -r network_key_b64' \ + 'read -r private_key_b64' \ + 'password=$(printf "%s" "$password_b64" | base64 -d)' \ + 'if [ "$3" = new ]; then' \ + ' interfold password set --config "$2"' \ ' mkdir -p "$(dirname "$PASSWORD_FILE")"' \ ' printf "%s" "$password" > "$PASSWORD_FILE"' \ ' chmod 400 "$PASSWORD_FILE"' \ 'fi' \ - '[ "$operation" != unexpected ]' \ - > "$bin_dir/interfold" - chmod +x "$bin_dir/interfold" + 'interfold net keypair set --config "$2"' \ + 'interfold wallet set --config "$2"' \ + 'unset password password_b64 network_key_b64 private_key_b64' \ + > "$bin_dir/expect" + chmod +x "$bin_dir/expect" } write_secrets() { @@ -67,7 +81,9 @@ run_entrypoint() { shift mkdir -p "$case_dir/data" "$case_dir/secrets" "$case_dir/bin" : > "$case_dir/calls" + : > "$case_dir/argv" make_mock_interfold "$case_dir/bin" + make_mock_expect "$case_dir/bin" env -u ENCRYPTION_PASSWORD -u NETWORK_PRIVATE_KEY -u PRIVATE_KEY \ PATH="$case_dir/bin:$PATH" \ @@ -75,8 +91,10 @@ run_entrypoint() { CONFIG_FILE="$case_dir/data/config.yaml" \ TEMPLATE_FILE="$ROOT_DIR/config.template.yaml" \ SECRETS_FILE="$case_dir/secrets/secrets.json" \ + CREDENTIAL_PROVISIONER="$ROOT_DIR/provision-credentials.exp" \ PASSWORD_FILE="$case_dir/data/password" \ CALL_LOG="$case_dir/calls" \ + ARGV_LOG="$case_dir/argv" \ RPC_URL="ws://127.0.0.1:8545" \ NODE_ADDRESS="0x3333333333333333333333333333333333333333" \ INTERFOLD_CONTRACT="0x4444444444444444444444444444444444444444" \ @@ -97,6 +115,9 @@ write_secrets "$success_dir/secrets/secrets.json" run_entrypoint "$success_dir" [ ! -e "$success_dir/secrets/secrets.json" ] || fail "successful setup retained plaintext credentials" [ "$(tr '\n' ' ' < "$success_dir/calls")" = "password network wallet start " ] || fail "unexpected successful command order" +assert_not_contains "$success_dir/argv" 'correct horse battery staple' +assert_not_contains "$success_dir/argv" '0x1111111111111111111111111111111111111111111111111111111111111111' +assert_not_contains "$success_dir/argv" '0x2222222222222222222222222222222222222222222222222222222222222222' assert_contains "$success_dir/data/config.yaml" 'autopassword: false' assert_contains "$success_dir/data/config.yaml" 'autonetkey: false' assert_contains "$success_dir/data/config.yaml" 'autowallet: false' @@ -154,6 +175,12 @@ if TEST_PRIVATE_KEY=legacy run_entrypoint "$legacy_dir"; then fail "legacy secret environment variable was accepted" fi +# Entrypoint regressions: credential values must never be expanded into CLI +# flags, even if those legacy flags remain available for interactive users. +assert_not_contains "$ROOT_DIR/entrypoint.sh" '--password "$password"' +assert_not_contains "$ROOT_DIR/entrypoint.sh" '--private-key "$private_key"' +assert_not_contains "$ROOT_DIR/entrypoint.sh" '--net-keypair "$network_private_key"' + # Health probe regression: require the exact process/config, protected files, # and bound QUIC listener rather than accepting an arbitrary matching PID. health_dir="$TEST_ROOT/health" From 762fccb56f8129ac37535da48a1058fec1efd8c9 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 14 Jul 2026 22:58:25 +0500 Subject: [PATCH 61/62] fix(storage): recover only uncommitted event-log tails --- agent/CRATES_ARCHITECTURE.md | 30 +- agent/flow-trace/00_INDEX.md | 1 + .../06_DEACTIVATION_AND_COMPLETION.md | 11 + crates/ciphernode-builder/src/event_system.rs | 20 +- crates/cli/src/events.rs | 2 +- crates/cli/src/node.rs | 15 +- crates/dashboard/src/lib.rs | 2 +- crates/data/src/commit_log_event_log.rs | 558 ++++++++++++++++-- crates/data/src/in_mem_event_log.rs | 21 +- crates/entrypoint/src/validate.rs | 71 ++- crates/events/src/events.rs | 18 +- crates/events/src/eventstore.rs | 211 ++++--- crates/events/src/eventstore_router.rs | 18 +- crates/events/src/traits.rs | 9 +- crates/net/src/network_sync/handlers.rs | 35 +- crates/sync/src/replay_spool.rs | 6 +- crates/sync/src/sync/preflight.rs | 1 + 17 files changed, 848 insertions(+), 181 deletions(-) diff --git a/agent/CRATES_ARCHITECTURE.md b/agent/CRATES_ARCHITECTURE.md index 333b85f23d..5fa69bb210 100644 --- a/agent/CRATES_ARCHITECTURE.md +++ b/agent/CRATES_ARCHITECTURE.md @@ -364,9 +364,13 @@ flowchart TD ``` The append-only event log is the durable source of truth. The timestamp index and -snapshots are derived state. EventStore construction reconciles missing index rows from -the log in strict 1,024-record pages and fail-stops on a gap or unusable index; it does -not decode the complete log solely to rebuild the index. +snapshots are derived state. Before `commitlog` opens, startup validates the active +segment's physical frames against its index. A CRC/length-invalid suffix after the last +indexed record is an uncommitted crash tail and is truncated; complete CRC-valid, +decodable frames whose index writes were lost are re-indexed. Indexed decode/CRC +failures and any offset/index mismatch remain fatal. EventStore construction then +performs a full integrity scan and reconciles missing timestamp-index rows from the log +in strict 1,024-record pages. Timestamp admission deduplicates by stable event ID plus payload, so the same logical event may return through historical network sync with a different transport source without colliding. A different payload at an already-indexed HLC timestamp remains an integrity failure. @@ -387,6 +391,9 @@ one burst, and EventBus subscriber fanout no longer bypasses downstream mailbox EventStore query responses also await recipient capacity, preventing a full aggregation mailbox from dropping one aggregate response and hanging startup. Recovery publishes `EffectsEnabled`, canonical history, and `SyncEnded` as three separately fenced phases. +Runtime log-read failures are returned in the correlated query response and flow through +the existing error paths; a remote sync query therefore cannot panic the EventStore +actor. The fail-stop behavior below applies to durable append/index-write failures. An event-log or timestamp-index write error panics the affected EventStore before live dispatch. This preserves durable-before-dispatch safety, but under the default unwind profile an Actix actor panic is contained at its spawned task boundary: it can kill the @@ -704,7 +711,9 @@ The whole barrier is time-bounded. Failure to drain or flush is returned to the and produces a non-zero exit. On restart, the process fence prevents two local writers from sharing one database. Schema preflight rejects unsupported upgrades or downgrades. `interfold node validate` provides offline integrity and loose-end -diagnostics; destructive automatic repair is not performed. +diagnostics without mutation by default. `interfold node validate --repair` is narrowly +allowed to perform the same safe uncommitted-tail recovery used at normal startup; it +never removes an indexed record. The implemented restart and operator-controlled recovery boundary is: @@ -712,7 +721,10 @@ The implemented restart and operator-controlled recovery boundary is: flowchart TD Incident[unclean exit, corruption warning, or unsupported schema] --> Stop[stop the node and preserve its data] Stop --> Validate[run interfold node validate offline] - Validate --> Decision{event log and schema usable?} + Validate --> Tail{recoverable uncommitted log tail?} + Tail -->|yes| Repair[run node validate --repair or start normally] + Repair --> Validate + Tail -->|no| Decision{event log and schema usable?} Decision -->|yes| Restart[normal node start] Restart --> Preflight[schema preflight and bounded index reconciliation] Preflight --> Replay[local snapshot plus event-log replay] @@ -732,9 +744,11 @@ flowchart TD class Blocked,Reset,Empty residual ``` -There is no in-process repair, event-log rollback, backup/restore command, or dedicated -full-resync command in the Rust crates. Backup restore is an offline filesystem -operation. A destructive reset removes the local event log—the node's source of +There is no rollback of indexed event records, backup/restore command, or dedicated +full-resync command in the Rust crates. The only automatic repair truncates bytes after +the last index boundary and restores complete CRC-valid/decodable frames whose index +entries were lost; committed corruption still fails closed. Backup restore is an offline +filesystem operation. A destructive reset removes the local event log—the node's source of truth—and can reconstruct only observations still available from configured EVM ranges and peers. One historical network startup attempt, including all aggregates and retries, is capped at 512 pages, 50,000 events, 128 MiB, and five minutes, with no diff --git a/agent/flow-trace/00_INDEX.md b/agent/flow-trace/00_INDEX.md index 907bed04ab..945db74f54 100644 --- a/agent/flow-trace/00_INDEX.md +++ b/agent/flow-trace/00_INDEX.md @@ -202,3 +202,4 @@ _Found during source-code cross-referencing of these trace documents._ | 13 | **Forward-compatible E3 version skew** | Resolved | A well-formed `E3Requested` carrying an unsupported committee-size or BFV-preset enum is now classified as a benign skip for that older binary. The parser emits an internal `Processed` marker so historical ordering and startup can advance without dispatching the E3. ABI decode failures and missing indexed topics remain rejected and fail the chain pipeline closed. | | 14 | **Post-startup network lag recovery** | Resolved | `NetEventBuffer` still fails readiness closed if its broadcast input lags during startup buffering, because historical/live reconciliation would be incomplete. Once `SyncEnded` has moved it to `Running`, a `Lagged(n)` signal is warned and skipped, and the ingress receive loop continues from the oldest retained event instead of stopping the buffer and permanently deafening gossip consumers. | | 15 | **Local/network event redelivery collision** | Resolved | EventStore now treats the same HLC timestamp plus stable event ID and equal payload as an idempotent duplicate even when transport context differs (`Local` versus `Net`). It retains the first stored context and still fails closed when different payloads claim the same timestamp, preserving collision detection without panicking on sync/resync redelivery. | +| 16 | **Crash-torn event-log tail** | Resolved | Startup validates active-segment frames against the commitlog index, truncates only an unindexed CRC/length-invalid physical suffix, and restores complete CRC-valid/decodable records whose tail index entry was lost. Indexed corruption remains fatal. Runtime reads return correlated errors instead of panicking query handlers, and `node validate --repair` exposes the same narrowly bounded recovery explicitly. | diff --git a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md index 4cb5fbc88a..fdaed70aea 100644 --- a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md +++ b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md @@ -216,6 +216,11 @@ interfold start → running node └─ Flushes the optional operational JSON log collector On restart: +├─ Event-log open: +│ → validates physical frames against the commitlog index +│ → truncates only a CRC/length-invalid suffix after the final indexed record +│ → restores complete CRC-valid, decodable frames whose tail index write was lost +│ → rejects indexed corruption, decode failure, gaps, and offset mismatches ├─ Sync module replays: │ 1. Load snapshot metadata and hydrate persisted per-E3 state │ → Extensions must preserve hydrated recipients; replayed committee events @@ -276,6 +281,12 @@ accept within the timeout fails recovery instead of being silently skipped. Snap contains asynchronous edges, so this does not claim that every downstream actor is synchronously durable at each replay step. +`interfold node validate` detects a recoverable uncommitted event-log tail without changing it. +With the node stopped, `interfold node validate --repair` applies the same boundary-checked tail +recovery as startup and refuses to remove indexed records. Runtime EventStore query failures are +returned to the correlated caller rather than panicking the actor; committed corruption remains a +startup/integrity failure. + ### Restart + Persist State Diagram ```mermaid diff --git a/crates/ciphernode-builder/src/event_system.rs b/crates/ciphernode-builder/src/event_system.rs index b524d594a9..cf61efd4bf 100644 --- a/crates/ciphernode-builder/src/event_system.rs +++ b/crates/ciphernode-builder/src/event_system.rs @@ -8,7 +8,7 @@ use crate::get_interfold_event_bus; use crate::global_eventstore_cache::{share_eventstore_reader, EventStoreReader}; use crate::global_store_cache::share_store; use actix::{Actor, Addr, Handler, Recipient}; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context as _, Result}; use e3_data::{ CommitLogEventLog, DataStore, InMemEventLog, InMemSequenceIndex, InMemStore, SledSequenceIndex, SledStore, @@ -264,6 +264,7 @@ impl EventSystem { InMemSequenceIndex::new(), InMemEventLog::new(), ) + .expect("in-memory EventStore reconciliation cannot fail") .start(), ); } @@ -287,8 +288,17 @@ impl EventSystem { let index_store = SledSequenceIndex::new(&b.sled_path, &tree_name)?; let log = CommitLogEventLog::new(&enumerated_log_path)?; - eventstore_map - .insert(index, EventStore::new(index_store, log).start()); + eventstore_map.insert( + index, + EventStore::new(index_store, log) + .with_context(|| { + format!( + "failed to reconcile EventStore aggregate \ + {index}" + ) + })? + .start(), + ); } Ok(eventstore_map) })? @@ -514,7 +524,9 @@ mod tests { impl Handler for Listener { type Result = (); fn handle(&mut self, msg: EventStoreQueryResponse, _: &mut Self::Context) -> Self::Result { - self.events = msg.into_events(); + self.events = msg + .into_events() + .expect("test EventStore query should succeed"); } } diff --git a/crates/cli/src/events.rs b/crates/cli/src/events.rs index a1e6cdfda5..3ee8dbb7eb 100644 --- a/crates/cli/src/events.rs +++ b/crates/cli/src/events.rs @@ -76,7 +76,7 @@ async fn fetch_events( .with_limit(limit); eventstore.seq().do_send(msg); - let events = rx.await?.into_events(); + let events = rx.await?.into_events()?; let next = compute_seq_cursor(&events, limit as usize); Ok((events, next)) diff --git a/crates/cli/src/node.rs b/crates/cli/src/node.rs index 22f5929c32..3d679cd732 100644 --- a/crates/cli/src/node.rs +++ b/crates/cli/src/node.rs @@ -17,20 +17,25 @@ pub enum NodeCommands { /// Takes the node's exclusive process fence and checks that the schema is /// loadable by this binary, the event log is intact, the snapshot cursor is /// consistent, and there are no orphaned committee tickets ("loose ends"). - /// Safe to run while the node is stopped; intended as the - /// pre-upgrade and post-crash health check. Exits non-zero on failure. - Validate, + /// Safe to run while the node is stopped; intended as the pre-upgrade and + /// post-crash health check. Exits non-zero on failure. Without `--repair`, + /// no files are changed. + Validate { + /// Repair only a provably uncommitted torn/unindexed event-log tail + #[arg(long)] + repair: bool, + }, } pub async fn execute(out: Console, command: NodeCommands, config: &AppConfig) -> Result<()> { match command { - NodeCommands::Validate => { + NodeCommands::Validate { repair } => { // Offline-only contract: hold the same cross-host fence `start` uses so the // validator cannot read state out from under a live node or race a concurrent // `interfold start`. Released when this scope ends. let _fence = e3_entrypoint::fence::ProcessFence::acquire(&config.db_file(), &config.name())?; - let report = validate_node(config).await?; + let report = validate_node(config, repair).await?; log!(out, "{}", report.render()); if report.has_failure() { bail!("node validation failed"); diff --git a/crates/dashboard/src/lib.rs b/crates/dashboard/src/lib.rs index f8d54d69c5..ee489a2c6d 100644 --- a/crates/dashboard/src/lib.rs +++ b/crates/dashboard/src/lib.rs @@ -158,7 +158,7 @@ impl DashboardState { let response = tokio::time::timeout(Duration::from_secs(5), response) .await .context("EventStore dashboard query timed out")??; - Ok(response.into_events()) + response.into_events() } async fn refresh_operator_status(&self) -> OperatorStatusSnapshot { diff --git a/crates/data/src/commit_log_event_log.rs b/crates/data/src/commit_log_event_log.rs index 8341ddb325..013d0e903d 100644 --- a/crates/data/src/commit_log_event_log.rs +++ b/crates/data/src/commit_log_event_log.rs @@ -5,25 +5,104 @@ // or FITNESS FOR A PARTICULAR PURPOSE. use anyhow::{anyhow, Context, Result}; -use commitlog::message::MessageSet; +use commitlog::message::{MessageBuf, MessageSet}; use commitlog::{CommitLog, LogOptions, ReadLimit}; use e3_events::{EventLog, InterfoldEvent, Unsequenced}; -use std::path::PathBuf; +use std::{ + collections::BTreeSet, + fs::{self, File, OpenOptions}, + io::{Read, Seek, SeekFrom, Write}, + path::{Path, PathBuf}, +}; +use tracing::warn; /// Maximum message size for both reads and writes (32 MB). const MAX_MESSAGE_BYTES: usize = 32 * 1024 * 1024; +const COMMITLOG_HEADER_BYTES: usize = 20; +const COMMITLOG_INDEX_ENTRY_BYTES: usize = 8; +const COMMITLOG_SEGMENT_MAGIC: [u8; 2] = [0xff, 0xff]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum EventLogOpenMode { + /// Repair only a provably uncommitted physical tail before opening. + RecoverTail, + /// Detect a recoverable tail without changing any on-disk bytes. + ValidateOnly, +} + +#[derive(Debug)] +struct TailRecoveryPlan { + segment_path: PathBuf, + indexed_end: u64, + physical_end: u64, + next_offset: u64, + recovered_payloads: Vec>, +} + +#[derive(Debug)] +enum PhysicalFrame { + Complete { + offset: u64, + payload: Vec, + end: u64, + }, + Incomplete(String), +} pub struct CommitLogEventLog { log: CommitLog, } impl CommitLogEventLog { - pub fn new(path: &PathBuf) -> Result { + pub fn new(path: &Path) -> Result { + Self::open(path, EventLogOpenMode::RecoverTail) + } + + pub fn open(path: &Path, mode: EventLogOpenMode) -> Result { + let recovery = inspect_active_tail(path)?; + if let Some(plan) = recovery.as_ref() { + match mode { + EventLogOpenMode::RecoverTail => apply_tail_recovery(plan)?, + EventLogOpenMode::ValidateOnly => { + let complete = plan.recovered_payloads.len(); + let tail_bytes = plan.physical_end.saturating_sub(plan.indexed_end); + anyhow::bail!( + "recoverable uncommitted event-log tail detected in {}: {complete} complete \ + unindexed record(s), {tail_bytes} physical tail byte(s); rerun with \ + `interfold node validate --repair` while the node is stopped", + plan.segment_path.display() + ); + } + } + } + let mut opts = LogOptions::new(path); // TODO: derive this from config - currently set high to be permissive opts.message_max_bytes(MAX_MESSAGE_BYTES); - let log = CommitLog::new(opts)?; - Ok(Self { log }) + let mut log = CommitLog::new(opts)?; + + if let Some(plan) = recovery { + for (index, payload) in plan.recovered_payloads.iter().enumerate() { + let expected_offset = plan.next_offset + index as u64; + let actual_offset = log + .append_msg(payload) + .context("failed to restore a complete unindexed event-log tail record")?; + if actual_offset != expected_offset { + anyhow::bail!( + "event-log tail recovery offset mismatch: expected {expected_offset}, got \ + {actual_offset}" + ); + } + } + log.flush() + .context("failed to flush repaired event-log tail")?; + } + + let opened = Self { log }; + opened + .read_from_checked(1) + .context("event log integrity check failed during open")?; + Ok(opened) } fn append_bytes(&mut self, bytes: &[u8]) -> Result { @@ -42,10 +121,8 @@ impl CommitLogEventLog { /// is not safe to treat as an ignorable torn tail. Its sequence number has /// already been committed, so skipping it would make replayed state diverge /// from the state that existed before the crash and a later append would turn - /// the same record into mid-log corruption. Callers that need a recoverable - /// integrity result (for example `interfold node validate`) use this method; - /// the [`EventLog`] adapter below fail-stops because its legacy iterator API - /// cannot carry an error. + /// the same record into mid-log corruption. The [`EventLog`] adapter carries + /// this error to startup and query callers instead of panicking an actor. pub fn read_from_checked(&self, from: u64) -> Result)>> { self.read_from_checked_with_limit(from, None) } @@ -77,6 +154,12 @@ impl CommitLogEventLog { let mut count = 0; for msg in message_buf.iter() { let seq = msg.offset() + 1; + if usize::from(msg.metadata_size()) > msg.size() as usize { + anyhow::bail!( + "commit log event at sequence {seq} has invalid frame metadata length; \ + log is corrupt" + ); + } let event = InterfoldEvent::::from_bytes(msg.payload()).with_context( || { format!( @@ -102,6 +185,292 @@ impl CommitLogEventLog { } } +fn inspect_active_tail(path: &Path) -> Result> { + if !path.exists() { + return Ok(None); + } + + let mut segments = BTreeSet::new(); + let mut indexes = BTreeSet::new(); + for entry in fs::read_dir(path) + .with_context(|| format!("failed to list event-log directory {}", path.display()))? + { + let entry = entry?; + if !entry.file_type()?.is_file() { + continue; + } + let file_path = entry.path(); + let Some(stem) = file_path.file_stem().and_then(|stem| stem.to_str()) else { + continue; + }; + if stem.len() != 20 || !stem.bytes().all(|byte| byte.is_ascii_digit()) { + continue; + } + let base_offset = stem.parse::().with_context(|| { + format!( + "invalid commit-log segment base offset in {}", + file_path.display() + ) + })?; + match file_path + .extension() + .and_then(|extension| extension.to_str()) + { + Some("log") => { + segments.insert(base_offset); + } + Some("index") => { + indexes.insert(base_offset); + } + _ => {} + } + } + + if segments != indexes { + anyhow::bail!( + "commit-log segment/index set mismatch in {} (segments: {segments:?}, indexes: \ + {indexes:?})", + path.display() + ); + } + let Some(base_offset) = segments.last().copied() else { + return Ok(None); + }; + + let segment_path = path.join(format!("{base_offset:020}.log")); + let index_path = path.join(format!("{base_offset:020}.index")); + let mut index_bytes = Vec::new(); + File::open(&index_path) + .with_context(|| format!("failed to open commit-log index {}", index_path.display()))? + .read_to_end(&mut index_bytes)?; + if index_bytes.len() % COMMITLOG_INDEX_ENTRY_BYTES != 0 { + anyhow::bail!( + "commit-log index {} has invalid length {}", + index_path.display(), + index_bytes.len() + ); + } + + let mut index_entries = Vec::new(); + let mut reached_empty = false; + for entry in index_bytes.chunks_exact(COMMITLOG_INDEX_ENTRY_BYTES) { + let relative_offset = u32::from_le_bytes(entry[0..4].try_into().unwrap()); + let file_position = u32::from_le_bytes(entry[4..8].try_into().unwrap()); + if relative_offset == 0 && file_position == 0 { + reached_empty = true; + continue; + } + if reached_empty { + anyhow::bail!( + "commit-log index {} contains a non-empty entry after its first empty slot", + index_path.display() + ); + } + let expected_relative = u32::try_from(index_entries.len()) + .context("commit-log index contains more than u32::MAX entries")?; + if relative_offset != expected_relative { + anyhow::bail!( + "commit-log index {} is non-contiguous at entry {}: expected relative offset {}, \ + got {}", + index_path.display(), + index_entries.len(), + expected_relative, + relative_offset + ); + } + index_entries.push(u64::from(file_position)); + } + + let mut segment = File::open(&segment_path).with_context(|| { + format!( + "failed to open commit-log segment {}", + segment_path.display() + ) + })?; + let physical_end = segment.metadata()?.len(); + let mut magic = [0u8; 2]; + segment.read_exact(&mut magic).with_context(|| { + format!( + "commit-log segment {} is missing its header", + segment_path.display() + ) + })?; + if magic != COMMITLOG_SEGMENT_MAGIC { + anyhow::bail!( + "commit-log segment {} has invalid version magic", + segment_path.display() + ); + } + + let mut indexed_end = COMMITLOG_SEGMENT_MAGIC.len() as u64; + for (index, position) in index_entries.iter().copied().enumerate() { + if position != indexed_end { + anyhow::bail!( + "commit-log index {} points entry {} at byte {}, expected byte {}", + index_path.display(), + index, + position, + indexed_end + ); + } + let expected_offset = base_offset + index as u64; + let expected_sequence = expected_offset + 1; + let PhysicalFrame::Complete { + offset, + payload, + end, + } = read_physical_frame(&mut segment, position, physical_end)? + else { + anyhow::bail!( + "committed event-log record at offset {expected_offset} is truncated or has an \ + invalid CRC" + ); + }; + if offset != expected_offset { + anyhow::bail!( + "committed event-log record offset mismatch: expected {expected_offset}, got \ + {offset}" + ); + } + InterfoldEvent::::from_bytes(&payload).with_context(|| { + format!("committed event-log record at sequence {expected_sequence} failed to decode") + })?; + indexed_end = end; + } + + let next_offset = base_offset + index_entries.len() as u64; + let mut position = indexed_end; + let mut recovered_payloads = Vec::new(); + while position < physical_end { + match read_physical_frame(&mut segment, position, physical_end)? { + PhysicalFrame::Complete { + offset, + payload, + end, + } => { + let expected_offset = next_offset + recovered_payloads.len() as u64; + if offset != expected_offset { + anyhow::bail!( + "unindexed event-log record offset mismatch at byte {position}: expected \ + {expected_offset}, got {offset}" + ); + } + InterfoldEvent::::from_bytes(&payload).with_context(|| { + format!( + "CRC-valid unindexed event-log record at offset {offset} failed to decode; \ + refusing to discard a potentially committed event" + ) + })?; + recovered_payloads.push(payload); + position = end; + } + PhysicalFrame::Incomplete(reason) => { + warn!( + segment = %segment_path.display(), + byte = position, + %reason, + "Detected an uncommitted torn event-log tail" + ); + break; + } + } + } + + if position == physical_end && recovered_payloads.is_empty() { + return Ok(None); + } + + Ok(Some(TailRecoveryPlan { + segment_path, + indexed_end, + physical_end, + next_offset, + recovered_payloads, + })) +} + +fn read_physical_frame(file: &mut File, position: u64, physical_end: u64) -> Result { + let header_end = position.saturating_add(COMMITLOG_HEADER_BYTES as u64); + if header_end > physical_end { + return Ok(PhysicalFrame::Incomplete(format!( + "only {} of {COMMITLOG_HEADER_BYTES} header bytes were written", + physical_end.saturating_sub(position) + ))); + } + + file.seek(SeekFrom::Start(position))?; + let mut header = [0u8; COMMITLOG_HEADER_BYTES]; + file.read_exact(&mut header)?; + let offset = u64::from_le_bytes(header[0..8].try_into().unwrap()); + let body_size = u32::from_le_bytes(header[8..12].try_into().unwrap()) as usize; + let metadata_size = u16::from_le_bytes(header[18..20].try_into().unwrap()) as usize; + if body_size > MAX_MESSAGE_BYTES || metadata_size > body_size { + return Ok(PhysicalFrame::Incomplete(format!( + "invalid frame sizes (body {body_size}, metadata {metadata_size})" + ))); + } + let frame_size = COMMITLOG_HEADER_BYTES + .checked_add(body_size) + .context("commit-log frame size overflow")?; + let end = position + .checked_add(frame_size as u64) + .context("commit-log frame position overflow")?; + if end > physical_end { + return Ok(PhysicalFrame::Incomplete(format!( + "frame declares {frame_size} bytes but only {} remain", + physical_end.saturating_sub(position) + ))); + } + + let mut frame = vec![0u8; frame_size]; + frame[..COMMITLOG_HEADER_BYTES].copy_from_slice(&header); + file.read_exact(&mut frame[COMMITLOG_HEADER_BYTES..])?; + let message_buf = match MessageBuf::from_bytes(frame) { + Ok(message_buf) => message_buf, + Err(error) => { + return Ok(PhysicalFrame::Incomplete(format!( + "frame CRC/length validation failed: {error:?}" + ))) + } + }; + let message = message_buf + .iter() + .next() + .context("commit-log frame contained no message")?; + Ok(PhysicalFrame::Complete { + offset, + payload: message.payload().to_vec(), + end, + }) +} + +fn apply_tail_recovery(plan: &TailRecoveryPlan) -> Result<()> { + let mut segment = OpenOptions::new() + .write(true) + .open(&plan.segment_path) + .with_context(|| { + format!( + "failed to open recoverable event-log tail {}", + plan.segment_path.display() + ) + })?; + segment.set_len(plan.indexed_end).with_context(|| { + format!( + "failed to truncate uncommitted event-log tail in {}", + plan.segment_path.display() + ) + })?; + segment.flush()?; + segment.sync_all()?; + warn!( + segment = %plan.segment_path.display(), + removed_bytes = plan.physical_end.saturating_sub(plan.indexed_end), + recovered_records = plan.recovered_payloads.len(), + "Recovered uncommitted event-log tail" + ); + Ok(()) +} + impl EventLog for CommitLogEventLog { fn append(&mut self, event: &InterfoldEvent) -> Result { let bytes = bincode::serialize(event)?; @@ -113,32 +482,22 @@ impl EventLog for CommitLogEventLog { Ok(()) } - fn read_from(&self, from: u64) -> Box)>> { - let events = self.read_from_checked(from).unwrap_or_else(|error| { - panic!( - "Event log integrity failure: {error:#}. Halting to prevent replay divergence; \ - run `interfold node validate` against the stopped node and restore from a \ - verified backup or resync." - ) - }); - Box::new(events.into_iter()) + fn read_from( + &self, + from: u64, + ) -> Result)>>> { + Ok(Box::new(self.read_from_checked(from)?.into_iter())) } fn read_from_bounded( &self, from: u64, limit: usize, - ) -> Box)>> { - let events = self - .read_from_checked_with_limit(from, Some(limit)) - .unwrap_or_else(|error| { - panic!( - "Event log integrity failure: {error:#}. Halting to prevent replay divergence; \ - run `interfold node validate` against the stopped node and restore from a \ - verified backup or resync." - ) - }); - Box::new(events.into_iter()) + ) -> Result)>>> { + Ok(Box::new( + self.read_from_checked_with_limit(from, Some(limit))? + .into_iter(), + )) } fn head(&self) -> u64 { @@ -151,6 +510,7 @@ impl EventLog for CommitLogEventLog { mod tests { use super::*; use e3_events::{EventConstructorWithTimestamp, EventSource, InterfoldEventData, TestEvent}; + use std::io::{Seek, SeekFrom, Write}; use tempfile::tempdir; // ── Event size reporting ───────────────────────────────────────────────── @@ -448,7 +808,7 @@ mod tests { #[test] fn test_append_and_read() { let dir = tempdir().unwrap(); - let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + let mut log = CommitLogEventLog::new(dir.path()).unwrap(); let event1 = event_from(TestEvent::new("one", 1)); let event2 = event_from(TestEvent::new("two", 2)); @@ -460,7 +820,7 @@ mod tests { assert_eq!(offset2, 2); // Read back from the beginning - let events: Vec<_> = log.read_from(1).collect(); + let events: Vec<_> = log.read_from(1).unwrap().collect(); assert_eq!(events.len(), 2); assert_eq!(events[0].0, 1); assert_eq!(events[1].0, 2); @@ -469,7 +829,7 @@ mod tests { #[test] fn test_read_from_offset() { let dir = tempdir().unwrap(); - let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + let mut log = CommitLogEventLog::new(dir.path()).unwrap(); let event1 = event_from(TestEvent::new("one", 1)); let event2 = event_from(TestEvent::new("two", 2)); @@ -480,7 +840,7 @@ mod tests { log.append(&event3).unwrap(); // Read from offset 2 (should get events 2 and 3) - let events: Vec<_> = log.read_from(2).collect(); + let events: Vec<_> = log.read_from(2).unwrap().collect(); assert_eq!(events.len(), 2); assert_eq!(events[0].0, 2); assert_eq!(events[1].0, 3); @@ -489,13 +849,13 @@ mod tests { #[test] fn bounded_read_decodes_only_requested_window() { let dir = tempdir().unwrap(); - let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + let mut log = CommitLogEventLog::new(dir.path()).unwrap(); for value in 1..=5 { log.append(&event_from(TestEvent::new("event", value))) .unwrap(); } - let events: Vec<_> = log.read_from_bounded(2, 2).collect(); + let events: Vec<_> = log.read_from_bounded(2, 2).unwrap().collect(); assert_eq!(events.len(), 2); assert_eq!(events[0].0, 2); assert_eq!(events[1].0, 3); @@ -504,7 +864,7 @@ mod tests { #[test] fn read_from_checked_reports_tail_corruption() { let dir = tempdir().unwrap(); - let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + let mut log = CommitLogEventLog::new(dir.path()).unwrap(); for i in 0..100 { let e = event_from(TestEvent::new("myevent", i)); @@ -520,22 +880,121 @@ mod tests { } #[test] - #[should_panic(expected = "Event log integrity failure")] - fn event_log_adapter_fail_stops_on_tail_corruption() { + fn startup_truncates_a_torn_unindexed_tail_and_remains_appendable() { + let dir = tempdir().unwrap(); + let path = dir.path().to_path_buf(); + let segment_path = path.join("00000000000000000000.log"); + let mut log = CommitLogEventLog::new(&path).unwrap(); + log.append(&event_from(TestEvent::new("before-crash", 1))) + .unwrap(); + log.flush().unwrap(); + drop(log); + + let indexed_len = fs::metadata(&segment_path).unwrap().len(); + OpenOptions::new() + .append(true) + .open(&segment_path) + .unwrap() + .write_all(b"partial-frame") + .unwrap(); + + let error = CommitLogEventLog::open(&path, EventLogOpenMode::ValidateOnly) + .err() + .unwrap(); + assert!( + format!("{error:#}").contains("recoverable uncommitted event-log tail"), + "{error:#}" + ); + assert_eq!( + fs::metadata(&segment_path).unwrap().len(), + indexed_len + b"partial-frame".len() as u64, + "detection-only validation must not mutate the segment" + ); + + let mut recovered = CommitLogEventLog::new(&path).unwrap(); + assert_eq!(fs::metadata(&segment_path).unwrap().len(), indexed_len); + assert_eq!(recovered.read_from(1).unwrap().count(), 1); + recovered + .append(&event_from(TestEvent::new("after-recovery", 2))) + .unwrap(); + assert_eq!(recovered.head(), 2); + assert_eq!(recovered.read_from(1).unwrap().count(), 2); + } + + #[test] + fn startup_reindexes_complete_crc_valid_tail_records() { + let dir = tempdir().unwrap(); + let path = dir.path().to_path_buf(); + let index_path = path.join("00000000000000000000.index"); + let mut log = CommitLogEventLog::new(&path).unwrap(); + log.append(&event_from(TestEvent::new("indexed", 1))) + .unwrap(); + log.append(&event_from(TestEvent::new("index-lost", 2))) + .unwrap(); + log.flush().unwrap(); + drop(log); + + let mut index = OpenOptions::new().write(true).open(&index_path).unwrap(); + index.seek(SeekFrom::Start(8)).unwrap(); + index.write_all(&[0u8; 8]).unwrap(); + index.sync_all().unwrap(); + drop(index); + + let error = CommitLogEventLog::open(&path, EventLogOpenMode::ValidateOnly) + .err() + .unwrap(); + assert!(format!("{error:#}").contains("1 complete unindexed record")); + + let recovered = CommitLogEventLog::new(&path).unwrap(); + let events: Vec<_> = recovered.read_from(1).unwrap().collect(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].0, 1); + assert_eq!(events[1].0, 2); + assert_eq!(recovered.head(), 2); + } + + #[test] + fn startup_refuses_crc_corruption_in_an_indexed_record() { + let dir = tempdir().unwrap(); + let path = dir.path().to_path_buf(); + let segment_path = path.join("00000000000000000000.log"); + let mut log = CommitLogEventLog::new(&path).unwrap(); + log.append(&event_from(TestEvent::new("committed", 1))) + .unwrap(); + log.flush().unwrap(); + drop(log); + + let mut segment = OpenOptions::new().write(true).open(&segment_path).unwrap(); + segment + .seek(SeekFrom::Start( + (COMMITLOG_SEGMENT_MAGIC.len() + COMMITLOG_HEADER_BYTES) as u64, + )) + .unwrap(); + segment.write_all(&[0xff]).unwrap(); + segment.sync_all().unwrap(); + + let error = CommitLogEventLog::new(&path).err().unwrap(); + let message = format!("{error:#}"); + assert!(message.contains("committed event-log record"), "{message}"); + assert!(message.contains("invalid CRC"), "{message}"); + } + + #[test] + fn event_log_adapter_returns_tail_corruption() { let dir = tempdir().unwrap(); - let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + let mut log = CommitLogEventLog::new(dir.path()).unwrap(); log.append(&event_from(TestEvent::new("valid", 1))).unwrap(); log.append_bytes(b"I am a bad event!").unwrap(); - let _: Vec<_> = log.read_from(1).collect(); + let error = log.read_from(1).err().unwrap(); + assert!(format!("{error:#}").contains("sequence 2")); } #[test] - #[should_panic(expected = "Event log integrity failure")] - fn test_read_from_non_tail_corruption_halts() { + fn test_read_from_non_tail_corruption_returns_error() { let dir = tempdir().unwrap(); - let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + let mut log = CommitLogEventLog::new(dir.path()).unwrap(); for i in 0..10 { let e = event_from(TestEvent::new("before", i)); @@ -549,13 +1008,14 @@ mod tests { log.append(&e).unwrap(); } - let _: Vec<_> = log.read_from(1).collect(); + let error = log.read_from(1).err().unwrap(); + assert!(format!("{error:#}").contains("failed to decode")); } #[test] fn test_head_reports_last_sequence() { let dir = tempdir().unwrap(); - let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + let mut log = CommitLogEventLog::new(dir.path()).unwrap(); assert_eq!(log.head(), 0); log.append(&event_from(TestEvent::new("one", 1))).unwrap(); log.append(&event_from(TestEvent::new("two", 2))).unwrap(); @@ -565,22 +1025,22 @@ mod tests { #[test] fn test_read_empty_log() { let dir = tempdir().unwrap(); - let log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + let log = CommitLogEventLog::new(dir.path()).unwrap(); - let events: Vec<_> = log.read_from(1).collect(); + let events: Vec<_> = log.read_from(1).unwrap().collect(); assert!(events.is_empty()); } #[test] fn test_read_past_end() { let dir = tempdir().unwrap(); - let mut log = CommitLogEventLog::new(&dir.path().to_path_buf()).unwrap(); + let mut log = CommitLogEventLog::new(dir.path()).unwrap(); let event = event_from(TestEvent::new("one", 1)); log.append(&event).unwrap(); // Read from offset beyond what exists - let events: Vec<_> = log.read_from(100).collect(); + let events: Vec<_> = log.read_from(100).unwrap().collect(); assert!(events.is_empty()); } } diff --git a/crates/data/src/in_mem_event_log.rs b/crates/data/src/in_mem_event_log.rs index 571b8e76c3..d714916c56 100644 --- a/crates/data/src/in_mem_event_log.rs +++ b/crates/data/src/in_mem_event_log.rs @@ -24,7 +24,10 @@ impl Default for InMemEventLog { } impl EventLog for InMemEventLog { - fn read_from(&self, from: u64) -> Box)>> { + fn read_from( + &self, + from: u64, + ) -> Result)>>> { // Convert 1-indexed sequence to 0-indexed array position let start_idx = from.saturating_sub(1) as usize; @@ -35,13 +38,13 @@ impl EventLog for InMemEventLog { .enumerate() .map(|(i, event)| (from + i as u64, event.clone())) .collect(); - Box::new(events.into_iter()) + Ok(Box::new(events.into_iter())) } fn read_from_bounded( &self, from: u64, limit: usize, - ) -> Box)>> { + ) -> Result)>>> { // Convert 1-indexed sequence to 0-indexed array position and clone only the requested // window. This path is used by bounded remote historical-sync queries. let start_idx = from.saturating_sub(1) as usize; @@ -53,7 +56,7 @@ impl EventLog for InMemEventLog { .enumerate() .map(|(i, event)| (from + i as u64, event.clone())) .collect(); - Box::new(events.into_iter()) + Ok(Box::new(events.into_iter())) } fn append(&mut self, event: &InterfoldEvent) -> Result { self.log.push(event.to_owned()); @@ -93,7 +96,7 @@ mod tests { assert_eq!(offset2, 2); // Read back from the beginning - let events: Vec<_> = log.read_from(1).collect(); + let events: Vec<_> = log.read_from(1).unwrap().collect(); assert_eq!(events.len(), 2); assert_eq!(events[0].0, 1); assert_eq!(events[1].0, 2); @@ -112,7 +115,7 @@ mod tests { log.append(&event3).unwrap(); // Read from offset 2 (should get events 2 and 3) - let events: Vec<_> = log.read_from(2).collect(); + let events: Vec<_> = log.read_from(2).unwrap().collect(); assert_eq!(events.len(), 2); assert_eq!(events[0].0, 2); assert_eq!(events[1].0, 3); @@ -126,7 +129,7 @@ mod tests { .unwrap(); } - let events: Vec<_> = log.read_from_bounded(2, 2).collect(); + let events: Vec<_> = log.read_from_bounded(2, 2).unwrap().collect(); assert_eq!(events.len(), 2); assert_eq!(events[0].0, 2); assert_eq!(events[1].0, 3); @@ -136,7 +139,7 @@ mod tests { fn test_read_empty_log() { let log = InMemEventLog::new(); - let events: Vec<_> = log.read_from(1).collect(); + let events: Vec<_> = log.read_from(1).unwrap().collect(); assert!(events.is_empty()); } @@ -148,7 +151,7 @@ mod tests { log.append(&event).unwrap(); // Read from offset beyond what exists - let events: Vec<_> = log.read_from(100).collect(); + let events: Vec<_> = log.read_from(100).unwrap().collect(); assert!(events.is_empty()); } } diff --git a/crates/entrypoint/src/validate.rs b/crates/entrypoint/src/validate.rs index 9519b08277..223e14c90c 100644 --- a/crates/entrypoint/src/validate.rs +++ b/crates/entrypoint/src/validate.rs @@ -11,10 +11,10 @@ //! operator question: *"Is my on-disk state intact, internally consistent, free //! of loose ends, and will this binary be able to load it after an upgrade?"* //! -//! It is deliberately non-destructive. It never mutates protocol state, never -//! talks to the chain, and never starts the node. It is safe to run while the node is -//! stopped (the recommended pre-upgrade step) and surfaces problems as a -//! structured report with a non-zero exit on failure. +//! It never mutates protocol state, talks to the chain, or starts the node. By +//! default it is fully non-destructive. The explicit `--repair` mode may only +//! truncate a provably uncommitted physical event-log tail and rebuild index +//! entries for complete CRC-valid tail records. //! //! ## Checks performed //! @@ -34,7 +34,7 @@ use crate::helpers::datastore::get_repositories; use anyhow::{bail, Context, Result}; use e3_config::AppConfig; -use e3_data::{CommitLogEventLog, Repositories}; +use e3_data::{CommitLogEventLog, EventLogOpenMode, Repositories}; use e3_events::{ AggregateId, E3Stage, Event, EventContextAccessors, EventContextSeq, InterfoldEvent, InterfoldEventData, @@ -159,20 +159,19 @@ impl ValidationReport { /// Opens the persisted stores while holding the node's exclusive process fence. /// Returns the full report; callers decide how to surface it (the CLI prints it /// and exits non-zero on failure). -pub async fn validate_node(config: &AppConfig) -> Result { +pub async fn validate_node(config: &AppConfig, repair: bool) -> Result { let aggregate_ids = aggregate_ids(config); let mut report = ValidationReport::default(); // 1. Read the commit logs directly before starting any EventStore actor. The - // actor-facing iterator intentionally fail-stops on corruption; the checked - // reader used here lets the operator receive a structured validation report. + // checked reader lets the operator receive a structured validation report. let mut terminal_keys: HashSet = HashSet::new(); let mut total_events: u64 = 0; let mut events_by_aggregate = Vec::with_capacity(aggregate_ids.len()); let mut unreadable_logs = 0usize; for agg in &aggregate_ids { let path = enumerate_path(&config.log_file(), agg.to_usize()); - match read_event_log(&path, *agg) { + match read_event_log(&path, *agg, repair) { Ok(events) => { total_events += events.len() as u64; collect_terminal_keys(&events, &mut terminal_keys); @@ -489,12 +488,21 @@ fn collect_terminal_keys(events: &[InterfoldEvent], out: &mut HashSet) { /// Read one aggregate's source-of-truth commit log without creating an empty log /// as a side effect when the node has never persisted that aggregate. -fn read_event_log(path: &Path, expected_aggregate: AggregateId) -> Result> { +fn read_event_log( + path: &Path, + expected_aggregate: AggregateId, + repair: bool, +) -> Result> { if !path.exists() { return Ok(Vec::new()); } - let log = CommitLogEventLog::new(&path.to_path_buf()) + let mode = if repair { + EventLogOpenMode::RecoverTail + } else { + EventLogOpenMode::ValidateOnly + }; + let log = CommitLogEventLog::open(path, mode) .with_context(|| format!("failed to open commit log {}", path.display()))?; let events: Vec = log .read_from_checked(1) @@ -531,6 +539,7 @@ mod tests { use commitlog::{CommitLog, LogOptions}; use e3_events::{EventConstructorWithTimestamp, EventLog, EventSource, TestEvent, Unsequenced}; use e3_sortition::OpenCommittee; + use std::{fs::OpenOptions, io::Write}; use tempfile::tempdir; #[test] @@ -543,7 +552,7 @@ mod tests { .unwrap(); drop(raw_log); - let error = read_event_log(&log_path, AggregateId::new(0)).unwrap_err(); + let error = read_event_log(&log_path, AggregateId::new(0), false).unwrap_err(); let message = format!("{error:#}"); assert!(message.contains("sequence 1"), "{message}"); assert!(message.contains("failed to decode"), "{message}"); @@ -554,12 +563,46 @@ mod tests { let dir = tempdir().unwrap(); let log_path = dir.path().join("log.0"); - assert!(read_event_log(&log_path, AggregateId::new(0)) + assert!(read_event_log(&log_path, AggregateId::new(0), false) .unwrap() .is_empty()); assert!(!log_path.exists()); } + #[test] + fn validator_repair_recovers_only_an_uncommitted_physical_tail() { + let dir = tempdir().unwrap(); + let log_path = dir.path().join("log.0"); + let segment_path = log_path.join("00000000000000000000.log"); + let mut log = CommitLogEventLog::new(&log_path).unwrap(); + let event = InterfoldEvent::::new_with_timestamp( + TestEvent::new("valid", 1).into(), + None, + 1, + None, + EventSource::Local, + ); + log.append(&event).unwrap(); + log.flush().unwrap(); + drop(log); + OpenOptions::new() + .append(true) + .open(segment_path) + .unwrap() + .write_all(b"torn") + .unwrap(); + + let detection = format!( + "{:#}", + read_event_log(&log_path, AggregateId::new(0), false).unwrap_err() + ); + assert!(detection.contains("recoverable uncommitted event-log tail")); + + let events = read_event_log(&log_path, AggregateId::new(0), true).unwrap(); + assert_eq!(events.len(), 1); + assert_eq!(events[0].seq(), 1); + } + #[test] fn validator_reader_rejects_event_in_wrong_aggregate_log() { let dir = tempdir().unwrap(); @@ -575,7 +618,7 @@ mod tests { log.append(&aggregate_zero_event).unwrap(); drop(log); - let error = read_event_log(&log_path, AggregateId::new(1)).unwrap_err(); + let error = read_event_log(&log_path, AggregateId::new(1), false).unwrap_err(); let message = error.to_string(); assert!(message.contains("belongs to aggregate 0"), "{message}"); assert!(message.contains("for aggregate 1"), "{message}"); diff --git a/crates/events/src/events.rs b/crates/events/src/events.rs index c011a351ff..1dec80193b 100644 --- a/crates/events/src/events.rs +++ b/crates/events/src/events.rs @@ -41,16 +41,26 @@ impl StoreEventRequested { #[rtype("()")] pub struct EventStoreQueryResponse { id: CorrelationId, - events: Vec>, + result: std::result::Result>, String>, } impl EventStoreQueryResponse { pub fn new(id: CorrelationId, events: Vec) -> Self { - Self { id, events } + Self { + id, + result: Ok(events), + } + } + + pub fn from_result(id: CorrelationId, result: Result>) -> Self { + Self { + id, + result: result.map_err(|error| format!("{error:#}")), + } } - pub fn into_events(self) -> Vec { - self.events + pub fn into_events(self) -> Result> { + self.result.map_err(anyhow::Error::msg) } pub fn id(&self) -> CorrelationId { diff --git a/crates/events/src/eventstore.rs b/crates/events/src/eventstore.rs index f43e7334ae..a1e069955d 100644 --- a/crates/events/src/eventstore.rs +++ b/crates/events/src/eventstore.rs @@ -10,7 +10,7 @@ use crate::{ EventStoreQueryResponse, InterfoldEvent, Seq, SequenceIndex, Sequenced, Ts, Unsequenced, }; use actix::{Actor, AsyncContext, Handler, Recipient, WrapFuture}; -use anyhow::{bail, Result}; +use anyhow::{bail, Context as _, Result}; use tracing::{error, warn}; const INDEX_RECONCILE_PAGE_SIZE: usize = 1_024; @@ -31,7 +31,7 @@ impl EventStore { ) -> Result>> { let ts = event.ts(); if let Some(indexed_seq) = self.index.get(ts)? { - let Some((logged_seq, existing)) = self.log.read_from_bounded(indexed_seq, 1).next() + let Some((logged_seq, existing)) = self.log.read_from_bounded(indexed_seq, 1)?.next() else { bail!( "event index corruption at timestamp {ts}: sequence {indexed_seq} is missing \ @@ -110,8 +110,8 @@ impl EventStore { let events = match (filter.as_ref(), limit) { (None, Some(limit)) => self .log - .read_from_bounded(seq, usize::try_from(limit).unwrap_or(usize::MAX)), - _ => self.log.read_from(seq), + .read_from_bounded(seq, usize::try_from(limit).unwrap_or(usize::MAX))?, + _ => self.log.read_from(seq)?, }; let result = self.collect_events(events, filter, limit); Ok(result) @@ -123,7 +123,7 @@ impl EventStore { query: u64, filter: Option, limit: Option, - ) -> Vec> { + ) -> Result>> { // H7: the replay cursor must never point past the log head. The snapshot // cursor is committed atomically with its snapshot data, so a cursor ahead // of the log can only happen if the two unsynchronised flush timers @@ -131,22 +131,25 @@ impl EventStore { // Replaying an empty range past the gap would be silent divergence, so we // halt loudly. `query == head + 1` is the legitimate "fully caught up" case. let head = self.log.head(); - if query > head + 1 { - panic!( + let caught_up = head + .checked_add(1) + .context("event-log sequence overflow while checking replay cursor")?; + if query > caught_up { + bail!( "Replay cursor seq {query} is ahead of the event-log head {head}: the snapshot \ cursor references events the log does not contain (lost in a crash flush window). \ Halting; operator recovery required." ); } - self.collect_events(self.log.read_from(query), filter, limit) + Ok(self.collect_events(self.log.read_from(query)?, filter, limit)) } } impl EventStore { - pub fn new(index: I, log: L) -> Self { + pub fn new(index: I, log: L) -> Result { let mut store = Self { index, log }; - store.reconcile_index(); - store + store.reconcile_index()?; + Ok(store) } /// H5: the commitlog append and the ts→seq index insert are two non-atomic @@ -155,10 +158,10 @@ impl EventStore { /// subscriber cursor) silently miss it until reconciled. On startup we walk /// the log (the source of truth) and backfill any missing index rows so the /// derived index can never lag the log across a restart. - fn reconcile_index(&mut self) { + fn reconcile_index(&mut self) -> Result<()> { let head = self.log.head(); if head == 0 { - return; + return Ok(()); } let mut repaired = 0u64; let mut next_sequence = 1u64; @@ -167,16 +170,22 @@ impl EventStore { for (seq, event) in self .log .read_from_bounded(next_sequence, INDEX_RECONCILE_PAGE_SIZE) + .with_context(|| { + format!( + "event log integrity failure during index reconciliation at sequence \ + {next_sequence}" + ) + })? { if seq != next_sequence { - panic!( + bail!( "Event log integrity failure during index reconciliation: expected sequence \ {next_sequence}, got {seq}. Halting; run `interfold node validate` and \ recover from a verified backup or controlled resync." ); } if seq > head { - panic!( + bail!( "Event log changed during index reconciliation: captured head {head}, but \ bounded read returned sequence {seq}. Halting to avoid an inconsistent \ derived index." @@ -186,26 +195,32 @@ impl EventStore { let ts = event.ts(); match self.index.get(ts) { Ok(Some(_)) => {} - Ok(None) => match self.index.insert(ts, seq) { - Ok(()) => repaired += 1, - Err(error) => panic!( - "Failed to backfill event index at sequence {seq}: {error}. Halting to \ - avoid serving an incomplete timestamp index." - ), - }, - Err(error) => panic!( - "Failed to read event index at timestamp {ts}: {error}. Halting to avoid \ - serving an unverified timestamp index." - ), + Ok(None) => { + self.index.insert(ts, seq).with_context(|| { + format!( + "failed to backfill event index at sequence {seq}; refusing to \ + serve an incomplete timestamp index" + ) + })?; + repaired += 1; + } + Err(error) => { + return Err(error).with_context(|| { + format!( + "failed to read event index at timestamp {ts}; refusing to serve \ + an unverified timestamp index" + ) + }) + } } - next_sequence = seq.checked_add(1).unwrap_or_else(|| { - panic!("Event log sequence overflow while reconciling sequence {seq}") - }); + next_sequence = seq.checked_add(1).with_context(|| { + format!("event log sequence overflow while reconciling sequence {seq}") + })?; page_len += 1; } if page_len == 0 { - panic!( + bail!( "Event log integrity failure during index reconciliation: log head is {head}, \ but no event was returned for sequence {next_sequence}. Halting; run \ `interfold node validate` and recover from a verified backup or controlled \ @@ -213,7 +228,7 @@ impl EventStore { ); } if page_len > INDEX_RECONCILE_PAGE_SIZE { - panic!( + bail!( "Event log violated bounded-read contract during index reconciliation: \ returned {page_len} events for limit {INDEX_RECONCILE_PAGE_SIZE}." ); @@ -224,6 +239,7 @@ impl EventStore { "Reconciled event index on startup: backfilled {repaired} missing ts→seq entries" ); } + Ok(()) } } @@ -274,18 +290,12 @@ impl Handler> for EventStor let limit = msg.limit(); let filter = msg.filter().cloned(); let sender = msg.sender(); - let response = self - .query_by_ts(query, filter, limit) - .map(|events| EventStoreQueryResponse::new(id, events)); + let response = + EventStoreQueryResponse::from_result(id, self.query_by_ts(query, filter, limit)); ctx.wait( async move { - match response { - Ok(response) => { - if let Err(error) = deliver_query_response(sender, response).await { - error!(%error, "Event-store query recipient closed"); - } - } - Err(error) => error!(%error, "Event-store timestamp query failed"), + if let Err(error) = deliver_query_response(sender, response).await { + error!(%error, "Event-store query recipient closed"); } } .into_actor(self), @@ -301,8 +311,8 @@ impl Handler> for EventSto let limit = msg.limit(); let filter = msg.filter().cloned(); let sender = msg.sender(); - let evts = self.query_by_seq(query, filter, limit); - let response = EventStoreQueryResponse::new(id, evts); + let response = + EventStoreQueryResponse::from_result(id, self.query_by_seq(query, filter, limit)); ctx.wait( async move { if let Err(error) = deliver_query_response(sender, response).await { @@ -372,6 +382,7 @@ mod tests { bounded_read_limit: Option>, flushes: Option>, unbounded_read_calls: Option>, + fail_reads: bool, } impl MockLog { @@ -382,6 +393,7 @@ mod tests { bounded_read_limit: None, flushes: None, unbounded_read_calls: None, + fail_reads: false, } } @@ -392,6 +404,7 @@ mod tests { bounded_read_limit: Some(tracker), flushes: None, unbounded_read_calls: None, + fail_reads: false, } } @@ -402,6 +415,7 @@ mod tests { bounded_read_limit: None, flushes: Some(tracker), unbounded_read_calls: None, + fail_reads: false, } } @@ -417,6 +431,14 @@ mod tests { bounded_read_limit: Some(bounded_read_limit), flushes: None, unbounded_read_calls: Some(unbounded_read_calls), + fail_reads: false, + } + } + + fn failing_reads() -> Self { + Self { + fail_reads: true, + ..Self::new() } } } @@ -437,7 +459,10 @@ mod tests { fn read_from( &self, from: u64, - ) -> Box)>> { + ) -> Result)>>> { + if self.fail_reads { + bail!("simulated event-log integrity failure"); + } if let Some(calls) = &self.unbounded_read_calls { calls.fetch_add(1, Ordering::SeqCst); } @@ -448,14 +473,17 @@ mod tests { .filter(move |(i, _)| (*i as u64 + 1) >= from) .map(|(i, e)| (i as u64 + 1, e.clone())) .collect(); - Box::new(items.into_iter()) + Ok(Box::new(items.into_iter())) } fn read_from_bounded( &self, from: u64, limit: usize, - ) -> Box)>> { + ) -> Result)>>> { + if self.fail_reads { + bail!("simulated event-log integrity failure"); + } if let Some(calls) = &self.bounded_read_calls { calls.fetch_add(1, Ordering::SeqCst); } @@ -470,7 +498,7 @@ mod tests { .take(limit) .map(|(i, event)| (i as u64 + 1, event.clone())) .collect(); - Box::new(items.into_iter()) + Ok(Box::new(items.into_iter())) } fn head(&self) -> u64 { @@ -510,7 +538,7 @@ mod tests { } fn new_store() -> EventStore { - EventStore::new(MockIndex::new(), MockLog::new()) + EventStore::new(MockIndex::new(), MockLog::new()).unwrap() } fn populated_store(events: &[InterfoldEvent]) -> EventStore { @@ -554,7 +582,7 @@ mod tests { store.store_event(make_local_event(100)).unwrap(); store.store_event(make_local_event(200)).unwrap(); - let logged: Vec<_> = store.log.read_from(1).collect(); + let logged: Vec<_> = store.log.read_from(1).unwrap().collect(); assert_eq!(logged.len(), 2); } @@ -565,6 +593,7 @@ mod tests { MockIndex::new(), MockLog::with_flush_tracker(Arc::clone(&flushes)), ) + .unwrap() .start(); store.send(FlushEventStores).await??; @@ -644,6 +673,45 @@ mod tests { received_rx.recv().await.unwrap(); } + #[actix::test] + async fn query_read_failure_is_returned_without_panicking_the_actor() { + struct ResponseSink(Option>); + + impl Actor for ResponseSink { + type Context = actix::Context; + } + + impl Handler for ResponseSink { + type Result = (); + + fn handle( + &mut self, + msg: EventStoreQueryResponse, + _: &mut Self::Context, + ) -> Self::Result { + self.0.take().unwrap().send(msg).ok(); + } + } + + let store = EventStore::new(MockIndex::new(), MockLog::failing_reads()) + .unwrap() + .start(); + let (sender, receiver) = tokio::sync::oneshot::channel(); + let sink = ResponseSink(Some(sender)).start(); + store.do_send(EventStoreQueryBy::::new( + CorrelationId::new(), + 1, + sink.recipient(), + )); + + let response = receiver.await.unwrap(); + let error = response.into_events().unwrap_err(); + assert!(error + .to_string() + .contains("simulated event-log integrity failure")); + store.send(FlushEventStores).await.unwrap().unwrap(); + } + #[test] fn startup_index_reconciliation_reads_log_in_bounded_pages() { let event_count = INDEX_RECONCILE_PAGE_SIZE * 2 + 2; @@ -660,7 +728,7 @@ mod tests { Arc::clone(&unbounded_calls), ); - let store = EventStore::new(MockIndex::new(), log); + let store = EventStore::new(MockIndex::new(), log).unwrap(); assert_eq!(unbounded_calls.load(Ordering::SeqCst), 0); assert_eq!(bounded_calls.load(Ordering::SeqCst), 3); @@ -682,7 +750,7 @@ mod tests { assert!(result.is_none()); } - assert_eq!(store.log.read_from(1).count(), 1); + assert_eq!(store.log.read_from(1).unwrap().count(), 1); } #[test] @@ -694,9 +762,9 @@ mod tests { .store_event(make_network_event(100)) .unwrap() .is_none()); - let stored = store.log.read_from(1).next().unwrap().1; + let stored = store.log.read_from(1).unwrap().next().unwrap().1; assert_eq!(stored.source(), EventSource::Local); - assert_eq!(store.log.read_from(1).count(), 1); + assert_eq!(store.log.read_from(1).unwrap().count(), 1); } #[test] @@ -710,7 +778,7 @@ mod tests { let message = error.to_string(); assert!(message.contains("timestamp collision"), "{message}"); assert!(message.contains("sequence 1"), "{message}"); - assert_eq!(store.log.read_from(1).count(), 1); + assert_eq!(store.log.read_from(1).unwrap().count(), 1); } // =========================================================================== @@ -725,7 +793,7 @@ mod tests { make_local_event(300), ]); - let events = store.query_by_seq(1, None, None); + let events = store.query_by_seq(1, None, None).unwrap(); assert_eq!(events.len(), 3); } @@ -739,7 +807,7 @@ mod tests { make_local_event(400), ]); - let events = store.query_by_seq(3, None, None); + let events = store.query_by_seq(3, None, None).unwrap(); assert_eq!(events.len(), 2); } @@ -753,8 +821,9 @@ mod tests { make_network_event(400), ]); - let events = - store.query_by_seq(1, Some(EventStoreFilter::Source(EventSource::Local)), None); + let events = store + .query_by_seq(1, Some(EventStoreFilter::Source(EventSource::Local)), None) + .unwrap(); assert_eq!(events.len(), 2); for e in &events { @@ -772,7 +841,7 @@ mod tests { make_local_event(500), ]); - let events = store.query_by_seq(1, None, Some(2)); + let events = store.query_by_seq(1, None, Some(2)).unwrap(); assert_eq!(events.len(), 2); } @@ -787,11 +856,13 @@ mod tests { make_network_event(500), ]); - let events = store.query_by_seq( - 1, - Some(EventStoreFilter::Source(EventSource::Local)), - Some(2), - ); + let events = store + .query_by_seq( + 1, + Some(EventStoreFilter::Source(EventSource::Local)), + Some(2), + ) + .unwrap(); assert_eq!(events.len(), 2); for e in &events { @@ -803,7 +874,7 @@ mod tests { fn seq_query_on_empty_log_returns_empty() { let store = new_store(); - let events = store.query_by_seq(1, None, None); + let events = store.query_by_seq(1, None, None).unwrap(); assert!(events.is_empty()); } @@ -892,7 +963,7 @@ mod tests { fn ts_query_pushes_unfiltered_limit_into_event_log() { let observed_limit = Arc::new(AtomicUsize::new(0)); let log = MockLog::with_bounded_read_tracker(observed_limit.clone()); - let mut store = EventStore::new(MockIndex::new(), log); + let mut store = EventStore::new(MockIndex::new(), log).unwrap(); for timestamp in [100, 200, 300, 400] { store.store_event(make_local_event(timestamp)).unwrap(); } @@ -990,12 +1061,12 @@ mod tests { make_local_event(300), ]); - let page1 = store.query_by_seq(1, None, Some(2)); + let page1 = store.query_by_seq(1, None, Some(2)).unwrap(); assert_eq!(page1.len(), 2); assert_eq!(page1[0].seq(), 1); assert_eq!(page1[1].seq(), 2); - let page2 = store.query_by_seq(2, None, Some(2)); + let page2 = store.query_by_seq(2, None, Some(2)).unwrap(); assert_eq!(page2.len(), 2); assert_eq!(page2[0].seq(), 2); assert_eq!(page2[1].seq(), 3); @@ -1009,13 +1080,13 @@ mod tests { make_local_event(300), ]); - let page1 = store.query_by_seq(1, None, Some(2)); + let page1 = store.query_by_seq(1, None, Some(2)).unwrap(); assert_eq!(page1.len(), 2); let cursor_seq = page1.last().unwrap().seq(); assert_eq!(cursor_seq, 2); - let page2 = store.query_by_seq(cursor_seq, None, None); + let page2 = store.query_by_seq(cursor_seq, None, None).unwrap(); assert_eq!(page2.len(), 2); let total: Vec<_> = page1.iter().chain(page2.iter()).collect(); diff --git a/crates/events/src/eventstore_router.rs b/crates/events/src/eventstore_router.rs index a0ba826e40..3c72ae92ed 100644 --- a/crates/events/src/eventstore_router.rs +++ b/crates/events/src/eventstore_router.rs @@ -59,7 +59,23 @@ impl Handler for QueryAggregator { aggregate_id, self.pending.len() ); - self.collected_events.extend(msg.into_events()); + let events = match msg.into_events() { + Ok(events) => events, + Err(error) => { + error!( + %error, + ?aggregate_id, + "Aggregate EventStore query failed; forwarding the failure" + ); + self.sender.do_send(EventStoreQueryResponse::from_result( + self.parent_id, + Err(error), + )); + ctx.notify(Die); + return; + } + }; + self.collected_events.extend(events); if self.pending.is_empty() { debug!("All aggregates fulfilled, sending response"); diff --git a/crates/events/src/traits.rs b/crates/events/src/traits.rs index 8e198ca6f0..0f9ec1d482 100644 --- a/crates/events/src/traits.rs +++ b/crates/events/src/traits.rs @@ -185,7 +185,10 @@ pub trait EventLog: Unpin + 'static { Ok(()) } /// Read all events starting from the given sequence number (inclusive) - fn read_from(&self, from: u64) -> Box)>>; + fn read_from( + &self, + from: u64, + ) -> Result)>>>; /// Read at most `limit` events starting from the given sequence number (inclusive). /// /// Implementations backed by persistent storage should override this method so a bounded @@ -195,8 +198,8 @@ pub trait EventLog: Unpin + 'static { &self, from: u64, limit: usize, - ) -> Box)>> { - Box::new(self.read_from(from).take(limit)) + ) -> Result)>>> { + Ok(Box::new(self.read_from(from)?.take(limit))) } /// The 1-indexed sequence number of the last appended event, or `0` if the log is empty. fn head(&self) -> u64; diff --git a/crates/net/src/network_sync/handlers.rs b/crates/net/src/network_sync/handlers.rs index 19fdd3f394..a8891b70a6 100644 --- a/crates/net/src/network_sync/handlers.rs +++ b/crates/net/src/network_sync/handlers.rs @@ -150,20 +150,37 @@ impl Handler for NetSyncManager { impl Handler for NetSyncManager { type Result = (); fn handle(&mut self, msg: EventStoreQueryResponse, _: &mut Self::Context) -> Self::Result { - // Post-restart re-broadcast response (own forwardable artifacts) — handled separately from - // peer sync-request responses. - if self.rebroadcast_query_ids.remove(&msg.id()) { - self.handle_rebroadcast_response(msg.into_events()); - return; - } + let response_id = msg.id(); + let is_rebroadcast = self.rebroadcast_query_ids.remove(&response_id); trap(EType::Net, &self.bus.clone(), || { + let events = match msg.into_events() { + Ok(events) => events, + Err(error) => { + if !is_rebroadcast { + if let Some(pending) = self.requests.remove(&response_id) { + pending.responder.respond(ProtocolResponse::Error( + "historical sync storage unavailable".to_string(), + ))?; + } + } + return Err(error); + } + }; + + // Post-restart re-broadcast response (own forwardable artifacts) — handled separately from + // peer sync-request responses. + if is_rebroadcast { + self.handle_rebroadcast_response(events); + return Ok(()); + } + info!("Received response from eventstore."); - let Some(pending) = self.requests.remove(&msg.id()) else { - bail!("responder not found for {}", msg.id()); + let Some(pending) = self.requests.remove(&response_id) else { + bail!("responder not found for {response_id}"); }; let fetch_request: FetchEventsSince = pending.responder.try_request_into()?; - match build_sync_batch(msg.into_events(), &fetch_request) { + match build_sync_batch(events, &fetch_request) { SyncBatchOutcome::BadRequest(reason) => pending.responder.bad_request(reason)?, SyncBatchOutcome::Batch(batch) => pending.responder.ok(batch)?, } diff --git a/crates/sync/src/replay_spool.rs b/crates/sync/src/replay_spool.rs index 78545ece60..c5eb0cee3e 100644 --- a/crates/sync/src/replay_spool.rs +++ b/crates/sync/src/replay_spool.rs @@ -164,10 +164,10 @@ async fn query_page( ) .await .context("EventStore router stopped during paged replay")?; - Ok(rx - .await + rx.await .context("EventStore did not return a paged replay response")? - .into_events()) + .into_events() + .context("EventStore paged replay query failed") } fn event_order_key(event: &InterfoldEvent) -> (u128, AggregateId, u64) { diff --git a/crates/sync/src/sync/preflight.rs b/crates/sync/src/sync/preflight.rs index 1b9b246803..b3a75055e5 100644 --- a/crates/sync/src/sync/preflight.rs +++ b/crates/sync/src/sync/preflight.rs @@ -63,5 +63,6 @@ async fn event_logs_have_events( .await .context("event-store query stopped during schema preflight")? .into_events() + .context("event-store query failed during schema preflight")? .is_empty()) } From 4cca970c05f35910a7d840515e8007c56aab1d71 Mon Sep 17 00:00:00 2001 From: Hamza Khalid Date: Tue, 14 Jul 2026 23:04:21 +0500 Subject: [PATCH 62/62] fix(dappnode): bridge legacy state through schema v1 --- agent/CRATES_ARCHITECTURE.md | 4 + agent/flow-trace/00_INDEX.md | 1 + .../06_DEACTIVATION_AND_COMPLETION.md | 5 ++ dappnode/README.md | 34 ++++++--- dappnode/dappnode_package.json | 4 +- dappnode/docker-compose.yml | 4 +- dappnode/entrypoint.sh | 51 +++++++++---- dappnode/healthcheck.sh | 8 +- dappnode/provision-credentials.exp | 26 ++----- dappnode/setup-wizard.yml | 3 +- dappnode/tests/test-hardening.sh | 74 +++++++++++++++---- 11 files changed, 147 insertions(+), 67 deletions(-) diff --git a/agent/CRATES_ARCHITECTURE.md b/agent/CRATES_ARCHITECTURE.md index 5fa69bb210..44c2bd5810 100644 --- a/agent/CRATES_ARCHITECTURE.md +++ b/agent/CRATES_ARCHITECTURE.md @@ -249,6 +249,10 @@ identity pair as fresh. A partial identity, any additional unversioned key, any log without a marker, upgrades, and downgrades fail closed. This narrow exception is required because autowallet atomically creates the two bootstrap identities before the builder can stamp the schema; it does not let protocol state bypass compatibility checks. +The DAppNode v0.2.3 package is the explicit bridge for the previously shipped v0.1.8 +state: its entrypoint atomically moves `/data/.enclave` to `/data/.interfold`, and the +v0.2.3 release stamps schema version 1 before a later fail-closed binary is installed. +If both namespace roots exist, the bridge refuses to choose between them. ## Actor and message topology diff --git a/agent/flow-trace/00_INDEX.md b/agent/flow-trace/00_INDEX.md index 945db74f54..97a1e9597d 100644 --- a/agent/flow-trace/00_INDEX.md +++ b/agent/flow-trace/00_INDEX.md @@ -203,3 +203,4 @@ _Found during source-code cross-referencing of these trace documents._ | 14 | **Post-startup network lag recovery** | Resolved | `NetEventBuffer` still fails readiness closed if its broadcast input lags during startup buffering, because historical/live reconciliation would be incomplete. Once `SyncEnded` has moved it to `Running`, a `Lagged(n)` signal is warned and skipped, and the ingress receive loop continues from the oldest retained event instead of stopping the buffer and permanently deafening gossip consumers. | | 15 | **Local/network event redelivery collision** | Resolved | EventStore now treats the same HLC timestamp plus stable event ID and equal payload as an idempotent duplicate even when transport context differs (`Local` versus `Net`). It retains the first stored context and still fails closed when different payloads claim the same timestamp, preserving collision detection without panicking on sync/resync redelivery. | | 16 | **Crash-torn event-log tail** | Resolved | Startup validates active-segment frames against the commitlog index, truncates only an unindexed CRC/length-invalid physical suffix, and restores complete CRC-valid/decodable records whose tail index entry was lost. Indexed corruption remains fatal. Runtime reads return correlated errors instead of panicking query handlers, and `node validate --repair` exposes the same narrowly bounded recovery explicitly. | +| 17 | **DAppNode v0.1.8 schema upgrade** | Resolved | DAppNode v0.2.3 is shipped as the required compatibility bridge. Its entrypoint atomically renames the legacy `/data/.enclave` state root to `/data/.interfold` (and refuses ambiguous dual roots), preserves existing encrypted identities, and then relies on the v0.2.3 release's one-time schema-1 stamp. Later fail-closed binaries therefore see a proven marker instead of permanently rejecting the shipped v0.1.8 datastore. | diff --git a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md index fdaed70aea..939165853f 100644 --- a/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md +++ b/agent/flow-trace/06_DEACTIVATION_AND_COMPLETION.md @@ -287,6 +287,11 @@ recovery as startup and refuses to remove indexed records. Runtime EventStore qu returned to the correlated caller rather than panicking the actor; committed corruption remains a startup/integrity failure. +For DAppNode installations, package v0.2.3 is the mandatory bridge from the shipped v0.1.8 state. +It atomically moves the legacy `.enclave` custom-config root to `.interfold`, preserves the encrypted +operator/libp2p identity, and lets the v0.2.3 binary stamp schema version 1 before later binaries +enforce the marker. An ambiguous volume containing both roots fails closed. + ### Restart + Persist State Diagram ```mermaid diff --git a/dappnode/README.md b/dappnode/README.md index 8aece563a1..8424043a5d 100644 --- a/dappnode/README.md +++ b/dappnode/README.md @@ -155,25 +155,30 @@ block heights. ### Credentials file -Create a local JSON file containing all three credential values: +Create a local JSON file containing the password and operator key: ```json { "password": "a strong encryption password", - "private_key": "0x<64 hex characters>", - "network_private_key": "0x<64 hex characters>" + "private_key": "0x<64 hex characters>" } ``` Upload it in the setup wizard as **Ciphernode Credentials JSON**. DAppNode copies it to `/run/secrets/secrets.json` before starting the container. The entrypoint validates a maximum size of 16 KiB, required fields, and key encodings, then runs the exact commands supported by the pinned -Interfold v0.1.8 image. Any failed command aborts startup. Wallet and network keys are encrypted in -`/data`; v0.1.8 stores its password key there as a mode-`0400` file. After successful persistence, +Interfold v0.2.3 image. Any failed command aborts startup. The wallet command atomically derives and +stores both the Ethereum and libp2p identities. Both keys are encrypted in `/data`; v0.2.3 stores +its password key there as a mode-`0400` file. After successful persistence, the entrypoint removes the combined plaintext upload. Provisioning sends the secrets through the CLI's hidden TTY prompts over stdin; plaintext credentials are never placed in process arguments or container environment variables. +Legacy three-field files containing `network_private_key` are accepted for upgrade compatibility, +but v0.2.3 ignores that obsolete field on a fresh setup. When encrypted identity state already +exists, a matching upload is removed without rewriting either identity; a password mismatch fails +closed. + The Ethereum key must correspond to `NODE_ADDRESS`. Keep an encrypted offline backup of this JSON; do not paste its contents into package environment variables, `EXTRA_OPTS`, logs, or support bundles. On an ordinary restart, persisted credential state is reused from `/data` and the upload is @@ -208,9 +213,8 @@ At container startup, `entrypoint.sh`: - network name and RPC URL - contract addresses and deploy blocks -4. Validates the uploaded credential file and programs password, network key, and wallet key. The - pinned v0.1.8 network-key command is `interfold net keypair set`; an isolated Expect helper feeds - its hidden prompts from stdin and failures stop startup. +4. Validates the uploaded credential file and programs the password plus atomic wallet/network + identity. An isolated Expect helper feeds hidden prompts from stdin and failures stop startup. 5. Builds CLI args, including verbosity and `--peer` flags from `PEERS`. 6. Executes: @@ -223,12 +227,22 @@ The state and databases live under `/data` inside the container, which is backed that volume includes the password key, DAppNode backups of `/data` must be encrypted and access-controlled even though the wallet and network keys inside the volume are encrypted. +### Required v0.1.8 upgrade bridge + +DAppNode package v0.2.3 is the required bridge from the shipped v0.1.8 package to later binaries. +On its first start it atomically renames the custom-config state root from `/data/.enclave` to +`/data/.interfold`, refusing to proceed if both roots exist. The v0.2.3 binary then stamps schema +version 1 using its release-era compatibility behavior. Later fail-closed binaries can therefore +verify the marker instead of rejecting the old unversioned datastore. Do not skip this package when +upgrading an existing v0.1.8 node, and keep a verified backup of `/data` until the bridge has started +successfully. + ## Health semantics -Interfold v0.1.8 does not expose a local readiness endpoint. The package health check therefore uses +Interfold v0.2.3 does not expose a local readiness endpoint. The package health check therefore uses the strongest non-invasive local signals available in that release: PID 1 must be the expected `interfold start` command using `/data/config.yaml`, the protected config/password files must exist, -the v0.1.8 Sled/event-log directories must be initialized, and the configured QUIC UDP listener must +the v0.2.3 Sled/event-log directories must be initialized, and the configured QUIC UDP listener must be bound. This detects the old false-positive case where an unrelated process matched `pgrep`, as well as missing credentials, uninitialized persistence, and a dead network listener. diff --git a/dappnode/dappnode_package.json b/dappnode/dappnode_package.json index 2c766564d4..7fe20093f3 100644 --- a/dappnode/dappnode_package.json +++ b/dappnode/dappnode_package.json @@ -1,7 +1,7 @@ { "name": "interfold-ciphernode.public.dappnode.eth", - "version": "0.1.8", - "upstreamVersion": "0.1.8", + "version": "0.2.3", + "upstreamVersion": "0.2.3", "upstreamArg": "UPSTREAM_VERSION", "upstreamRepo": "theinterfold/interfold", "shortDescription": "Interfold Ciphernode - Run a node for confidential computing", diff --git a/dappnode/docker-compose.yml b/dappnode/docker-compose.yml index 1bc0cb4026..0dc9e0259f 100644 --- a/dappnode/docker-compose.yml +++ b/dappnode/docker-compose.yml @@ -4,8 +4,8 @@ services: build: context: . args: - UPSTREAM_VERSION: 0.1.8 - image: 'ciphernode.interfold-ciphernode.public.dappnode.eth:0.1.8' + UPSTREAM_VERSION: 0.2.3 + image: 'ciphernode.interfold-ciphernode.public.dappnode.eth:0.2.3' restart: unless-stopped # Allow the node's 30-second durability barrier to finish before Docker sends SIGKILL. stop_grace_period: 45s diff --git a/dappnode/entrypoint.sh b/dappnode/entrypoint.sh index 86fadb3caf..1c38de0b28 100644 --- a/dappnode/entrypoint.sh +++ b/dappnode/entrypoint.sh @@ -9,9 +9,11 @@ CONFIG_FILE="${CONFIG_FILE:-$CONFIG_DIR/config.yaml}" TEMPLATE_FILE="${TEMPLATE_FILE:-/opt/config.template.yaml}" SECRETS_FILE="${SECRETS_FILE:-/run/secrets/secrets.json}" CREDENTIAL_PROVISIONER="${CREDENTIAL_PROVISIONER:-/opt/provision-credentials.exp}" -# Interfold v0.1.8 resolves a relative `key_file: key` beside a discovered +LEGACY_STATE_DIR="${LEGACY_STATE_DIR:-$CONFIG_DIR/.enclave}" +CURRENT_STATE_DIR="${CURRENT_STATE_DIR:-$CONFIG_DIR/.interfold}" +# Interfold v0.2.3 resolves a relative `key_file: key` beside a discovered # /data/config.yaml to this path for the default node profile. -PASSWORD_FILE="${PASSWORD_FILE:-$CONFIG_DIR/.enclave/config/_default/key}" +PASSWORD_FILE="${PASSWORD_FILE:-$CURRENT_STATE_DIR/config/_default/key}" log() { printf '[%s] %s\n' "$(date '+%H:%M:%S')" "$1"; } fail() { @@ -36,6 +38,28 @@ fi [ -r "$TEMPLATE_FILE" ] || fail "configuration template is not readable: $TEMPLATE_FILE" mkdir -p "$CONFIG_DIR" +migrate_legacy_state() { + if [ -L "$LEGACY_STATE_DIR" ] || [ -L "$CURRENT_STATE_DIR" ]; then + fail "legacy/current state paths must not be symbolic links" + fi + if [ -e "$LEGACY_STATE_DIR" ] && [ ! -d "$LEGACY_STATE_DIR" ]; then + fail "legacy state path is not a directory: $LEGACY_STATE_DIR" + fi + if [ -e "$CURRENT_STATE_DIR" ] && [ ! -d "$CURRENT_STATE_DIR" ]; then + fail "current state path is not a directory: $CURRENT_STATE_DIR" + fi + if [ -d "$LEGACY_STATE_DIR" ] && [ -e "$CURRENT_STATE_DIR" ]; then + fail "both legacy and current state directories exist; refusing an ambiguous upgrade" + fi + if [ -d "$LEGACY_STATE_DIR" ]; then + log "Migrating the v0.1.8 state namespace to Interfold v0.2.3..." + mv -- "$LEGACY_STATE_DIR" "$CURRENT_STATE_DIR" \ + || fail "could not migrate legacy state into $CURRENT_STATE_DIR" + fi +} + +migrate_legacy_state + # Set non-secret defaults. export NETWORK="${NETWORK:-sepolia}" export QUIC_PORT="${QUIC_PORT:-37173}" @@ -64,12 +88,14 @@ validate_secret_file() { jq -e ' type == "object" and - (keys | sort == ["network_private_key", "password", "private_key"]) and + ((keys | sort == ["password", "private_key"]) or + (keys | sort == ["network_private_key", "password", "private_key"])) and (.password | type == "string" and length > 0 and length <= 1024 and test("^[^\\r\\n\\u0000]+$") and . == gsub("^\\s+|\\s+$"; "")) and (.private_key | type == "string" and test("^0x[0-9a-fA-F]{64}$")) and - (.network_private_key | type == "string" and test("^0x[0-9a-fA-F]{64}$")) - ' "$SECRETS_FILE" >/dev/null || fail "credentials file must contain valid password, private_key, and network_private_key strings" + ((has("network_private_key") | not) or + (.network_private_key | type == "string" and test("^0x[0-9a-fA-F]{64}$"))) + ' "$SECRETS_FILE" >/dev/null || fail "credentials file must contain valid password and private_key strings" } validate_persisted_password_file() { @@ -82,25 +108,24 @@ validate_persisted_password_file() { configure_credentials() { validate_secret_file - [ -r "$CREDENTIAL_PROVISIONER" ] || fail "credential provisioner is not readable: $CREDENTIAL_PROVISIONER" - - local provisioning_mode=new - if [ -e "$PASSWORD_FILE" ]; then validate_persisted_password_file jq -er '.password' "$SECRETS_FILE" | tr -d '\n' | cmp -s - "$PASSWORD_FILE" \ || fail "uploaded password does not match the persisted credential key" - provisioning_mode=existing log "Using the matching persisted encryption password." + rm -f "$SECRETS_FILE" + log "Existing encrypted wallet/network identity was preserved." + return fi + [ -r "$CREDENTIAL_PROVISIONER" ] || fail "credential provisioner is not readable: $CREDENTIAL_PROVISIONER" log "Provisioning encrypted credentials through hidden stdin prompts..." - jq -jr '[.password, .network_private_key, .private_key][] | @base64 + "\n"' "$SECRETS_FILE" \ - | expect "$CREDENTIAL_PROVISIONER" "$CONFIG_FILE" "$provisioning_mode" \ + jq -jr '[.password, .private_key][] | @base64 + "\n"' "$SECRETS_FILE" \ + | expect "$CREDENTIAL_PROVISIONER" "$CONFIG_FILE" \ || fail "one or more credential commands failed" # DAppNode copies fileUpload content into this container before startup. - # Wallet/network keys are encrypted in /data and v0.1.8 stores the password + # Wallet/network keys are encrypted in /data and v0.2.3 stores the password # key there with mode 0400. Remove the combined plaintext upload. rm -f "$SECRETS_FILE" log "Credential setup completed." diff --git a/dappnode/healthcheck.sh b/dappnode/healthcheck.sh index 83cd82faaf..eb977f8ffc 100644 --- a/dappnode/healthcheck.sh +++ b/dappnode/healthcheck.sh @@ -1,5 +1,5 @@ #!/bin/sh -# Best available local health signal for the v0.1.8 image. That release has no +# Best available local health signal for the v0.2.3 image. That release has no # readiness HTTP/control endpoint, so require the expected PID 1, its exact # start/config arguments, protected local credential/config files, and the QUIC # listener. This is stronger than PID-only liveness but is not a chain-sync or @@ -8,9 +8,9 @@ set -eu PROC_ROOT="${PROC_ROOT:-/proc}" CONFIG_FILE="${CONFIG_FILE:-/data/config.yaml}" -PASSWORD_FILE="${PASSWORD_FILE:-/data/.enclave/config/_default/key}" -DB_PATH="${DB_PATH:-/data/.enclave/data/_default/db}" -EVENT_LOG_PATH="${EVENT_LOG_PATH:-/data/.enclave/data/_default/log.0}" +PASSWORD_FILE="${PASSWORD_FILE:-/data/.interfold/config/_default/key}" +DB_PATH="${DB_PATH:-/data/.interfold/data/_default/db}" +EVENT_LOG_PATH="${EVENT_LOG_PATH:-/data/.interfold/data/_default/log.0}" QUIC_PORT="${QUIC_PORT:-37173}" SS_BIN="${SS_BIN:-ss}" STAT_BIN="${STAT_BIN:-stat}" diff --git a/dappnode/provision-credentials.exp b/dappnode/provision-credentials.exp index 75aa6fcaa6..10c738e904 100644 --- a/dappnode/provision-credentials.exp +++ b/dappnode/provision-credentials.exp @@ -1,6 +1,6 @@ #!/usr/bin/expect -f # Feed credentials to the hidden interactive prompts exposed by Interfold -# v0.1.8. Secrets arrive as base64-encoded lines on stdin; neither the +# v0.2.3. Secrets arrive as base64-encoded lines on stdin; neither the # plaintext nor the encoded value is placed in argv or the environment. set timeout 30 @@ -55,32 +55,20 @@ proc send_at_prompt {pattern secret description} { } } -if {$argc != 2} { - fail "expected CONFIG_FILE and provisioning mode" +if {$argc != 1} { + fail "expected CONFIG_FILE" } set config_file [lindex $argv 0] -set mode [lindex $argv 1] -if {$mode ne "new" && $mode ne "existing"} { - fail "provisioning mode must be new or existing" -} set password [read_secret "password"] -set network_private_key [read_secret "network private key"] set private_key [read_secret "wallet private key"] -if {$mode eq "new"} { - spawn -noecho interfold password set --config $config_file - send_at_prompt {Please enter a new password} $password "new password" - send_at_prompt {Please confirm your password} $password "password confirmation" - finish_child "password command" -} +spawn -noecho interfold password set --config $config_file +send_at_prompt {Please enter a new password} $password "new password" +send_at_prompt {Please confirm your password} $password "password confirmation" +finish_child "password command" unset password -spawn -noecho interfold net keypair set --config $config_file -send_at_prompt {Enter your network private key} $network_private_key "network private key" -finish_child "network key command" -unset network_private_key - spawn -noecho interfold wallet set --config $config_file send_at_prompt {Enter your Ethereum private key} $private_key "wallet private key" finish_child "wallet command" diff --git a/dappnode/setup-wizard.yml b/dappnode/setup-wizard.yml index c628cd4754..214bb8e51b 100644 --- a/dappnode/setup-wizard.yml +++ b/dappnode/setup-wizard.yml @@ -135,8 +135,7 @@ fields: ```json { "password": "a strong encryption password", - "private_key": "0x<64 hex characters>", - "network_private_key": "0x<64 hex characters>" + "private_key": "0x<64 hex characters>" } ``` diff --git a/dappnode/tests/test-hardening.sh b/dappnode/tests/test-hardening.sh index 497767c345..4bd5ca14a2 100644 --- a/dappnode/tests/test-hardening.sh +++ b/dappnode/tests/test-hardening.sh @@ -51,23 +51,27 @@ make_mock_expect() { '#!/bin/bash' \ 'set -Eeuo pipefail' \ 'read -r password_b64' \ - 'read -r network_key_b64' \ 'read -r private_key_b64' \ 'password=$(printf "%s" "$password_b64" | base64 -d)' \ - 'if [ "$3" = new ]; then' \ - ' interfold password set --config "$2"' \ - ' mkdir -p "$(dirname "$PASSWORD_FILE")"' \ - ' printf "%s" "$password" > "$PASSWORD_FILE"' \ - ' chmod 400 "$PASSWORD_FILE"' \ - 'fi' \ - 'interfold net keypair set --config "$2"' \ + 'interfold password set --config "$2"' \ + 'mkdir -p "$(dirname "$PASSWORD_FILE")"' \ + 'printf "%s" "$password" > "$PASSWORD_FILE"' \ + 'chmod 400 "$PASSWORD_FILE"' \ 'interfold wallet set --config "$2"' \ - 'unset password password_b64 network_key_b64 private_key_b64' \ + 'unset password password_b64 private_key_b64' \ > "$bin_dir/expect" chmod +x "$bin_dir/expect" } write_secrets() { + local path=$1 + printf '%s\n' '{' \ + ' "password": "correct horse battery staple",' \ + ' "private_key": "0x1111111111111111111111111111111111111111111111111111111111111111"' \ + '}' > "$path" +} + +write_legacy_secrets() { local path=$1 printf '%s\n' '{' \ ' "password": "correct horse battery staple",' \ @@ -107,17 +111,16 @@ run_entrypoint() { "$@" bash "$ROOT_DIR/entrypoint.sh" > "$case_dir/output" 2>&1 } -# Successful provisioning uses the exact v0.1.8 commands, removes the +# Successful provisioning uses the v0.2.3 atomic wallet command, removes the # plaintext upload, and starts only after every credential command succeeds. success_dir="$TEST_ROOT/success" mkdir -p "$success_dir/secrets" write_secrets "$success_dir/secrets/secrets.json" run_entrypoint "$success_dir" [ ! -e "$success_dir/secrets/secrets.json" ] || fail "successful setup retained plaintext credentials" -[ "$(tr '\n' ' ' < "$success_dir/calls")" = "password network wallet start " ] || fail "unexpected successful command order" +[ "$(tr '\n' ' ' < "$success_dir/calls")" = "password wallet start " ] || fail "unexpected successful command order" assert_not_contains "$success_dir/argv" 'correct horse battery staple' assert_not_contains "$success_dir/argv" '0x1111111111111111111111111111111111111111111111111111111111111111' -assert_not_contains "$success_dir/argv" '0x2222222222222222222222222222222222222222222222222222222222222222' assert_contains "$success_dir/data/config.yaml" 'autopassword: false' assert_contains "$success_dir/data/config.yaml" 'autonetkey: false' assert_contains "$success_dir/data/config.yaml" 'autowallet: false' @@ -126,10 +129,10 @@ assert_contains "$success_dir/data/config.yaml" 'autowallet: false' failure_dir="$TEST_ROOT/failure" mkdir -p "$failure_dir/secrets" write_secrets "$failure_dir/secrets/secrets.json" -if run_entrypoint "$failure_dir" FAIL_ON=network; then - fail "network credential failure was ignored" +if run_entrypoint "$failure_dir" FAIL_ON=wallet; then + fail "wallet credential failure was ignored" fi -assert_contains "$failure_dir/calls" 'network' +assert_contains "$failure_dir/calls" 'wallet' assert_not_contains "$failure_dir/calls" 'start' [ -e "$failure_dir/secrets/secrets.json" ] || fail "failed setup removed recovery input" @@ -143,6 +146,24 @@ if run_entrypoint "$mismatch_dir"; then fi [ ! -s "$mismatch_dir/calls" ] || fail "password mismatch mutated credentials" +# A matching upload on existing state must not rotate either persisted identity. +matching_dir="$TEST_ROOT/password-match" +mkdir -p "$matching_dir/data" "$matching_dir/secrets" +printf '%s' 'correct horse battery staple' > "$matching_dir/data/password" +write_secrets "$matching_dir/secrets/secrets.json" +run_entrypoint "$matching_dir" +[ "$(tr '\n' ' ' < "$matching_dir/calls")" = "start " ] || fail "matching persisted state was re-provisioned" +[ ! -e "$matching_dir/secrets/secrets.json" ] || fail "matching upload was not removed" + +# Legacy three-field uploads remain accepted; v0.2.3 derives the libp2p key +# atomically from the wallet key and ignores the obsolete separate network key. +legacy_credentials_dir="$TEST_ROOT/legacy-credentials" +mkdir -p "$legacy_credentials_dir/secrets" +write_legacy_secrets "$legacy_credentials_dir/secrets/secrets.json" +run_entrypoint "$legacy_credentials_dir" +[ "$(tr '\n' ' ' < "$legacy_credentials_dir/calls")" = "password wallet start " ] \ + || fail "legacy credential upload was not accepted" + # Malformed or absent first-start credentials fail before invoking Interfold. malformed_dir="$TEST_ROOT/malformed" mkdir -p "$malformed_dir/secrets" @@ -167,6 +188,26 @@ chmod 400 "$restart_dir/data/password" run_entrypoint "$restart_dir" [ "$(tr '\n' ' ' < "$restart_dir/calls")" = "start " ] || fail "persisted restart unexpectedly re-provisioned credentials" +# The 0.1.8 -> 0.2.3 bridge moves the complete custom-config namespace in one +# rename, preserving the unversioned DB/event log for v0.2.3 to stamp schema 1. +upgrade_dir="$TEST_ROOT/legacy-upgrade" +mkdir -p "$upgrade_dir/data/.enclave/config/_default" "$upgrade_dir/data/.enclave/data/_default/db" \ + "$upgrade_dir/data/.enclave/data/_default/log.0" +printf '%s' 'persisted-password' > "$upgrade_dir/data/.enclave/config/_default/key" +printf '%s' 'legacy-state' > "$upgrade_dir/data/.enclave/data/_default/db/sentinel" +run_entrypoint "$upgrade_dir" \ + PASSWORD_FILE="$upgrade_dir/data/.interfold/config/_default/key" +[ ! -e "$upgrade_dir/data/.enclave" ] || fail "legacy state namespace remained after upgrade" +assert_contains "$upgrade_dir/data/.interfold/data/_default/db/sentinel" 'legacy-state' +[ "$(tr '\n' ' ' < "$upgrade_dir/calls")" = "start " ] || fail "legacy state upgrade did not start" + +ambiguous_dir="$TEST_ROOT/ambiguous-upgrade" +mkdir -p "$ambiguous_dir/data/.enclave" "$ambiguous_dir/data/.interfold" +if run_entrypoint "$ambiguous_dir"; then + fail "ambiguous legacy/current state was accepted" +fi +[ ! -s "$ambiguous_dir/calls" ] || fail "ambiguous state invoked Interfold" + # Legacy secret environment variables are explicitly rejected. legacy_dir="$TEST_ROOT/legacy-env" mkdir -p "$legacy_dir/data" @@ -180,6 +221,9 @@ fi assert_not_contains "$ROOT_DIR/entrypoint.sh" '--password "$password"' assert_not_contains "$ROOT_DIR/entrypoint.sh" '--private-key "$private_key"' assert_not_contains "$ROOT_DIR/entrypoint.sh" '--net-keypair "$network_private_key"' +assert_contains "$ROOT_DIR/dappnode_package.json" '"version": "0.2.3"' +assert_contains "$ROOT_DIR/docker-compose.yml" 'UPSTREAM_VERSION: 0.2.3' +assert_contains "$ROOT_DIR/healthcheck.sh" '/data/.interfold/data/_default/db' # Health probe regression: require the exact process/config, protected files, # and bound QUIC listener rather than accepting an arbitrary matching PID.